diff options
Diffstat (limited to 'lib')
425 files changed, 3926 insertions, 3534 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index 8ebd7f83acb..8a26ae7e6f6 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -7,7 +7,7 @@ module API LOG_FILENAME = Rails.root.join("log", "api_json.log") - NO_SLASH_URL_PART_REGEX = %r{[^/]+}.freeze + NO_SLASH_URL_PART_REGEX = %r{[^/]+} NAMESPACE_OR_PROJECT_REQUIREMENTS = { id: NO_SLASH_URL_PART_REGEX }.freeze COMMIT_ENDPOINT_REQUIREMENTS = NAMESPACE_OR_PROJECT_REQUIREMENTS.merge(sha: NO_SLASH_URL_PART_REGEX).freeze USER_REQUIREMENTS = { user_id: NO_SLASH_URL_PART_REGEX }.freeze @@ -375,6 +375,7 @@ module API mount ::API::Todos mount ::API::UsageData mount ::API::UsageDataNonSqlMetrics + mount ::API::VsCode::Settings::VsCodeSettingsSync mount ::API::Ml::Mlflow::Entrypoint end diff --git a/lib/api/bulk_imports.rb b/lib/api/bulk_imports.rb index b4ace6cd6bc..9bcc16cf211 100644 --- a/lib/api/bulk_imports.rb +++ b/lib/api/bulk_imports.rb @@ -33,7 +33,8 @@ module API end before do - not_found! unless Gitlab::CurrentSettings.bulk_import_enabled? + not_found! unless Gitlab::CurrentSettings.bulk_import_enabled? || + Feature.enabled?(:override_bulk_import_disabled, current_user, type: :ops) authenticate! end diff --git a/lib/api/ci/helpers/runner.rb b/lib/api/ci/helpers/runner.rb index 542b2390df2..382528c814c 100644 --- a/lib/api/ci/helpers/runner.rb +++ b/lib/api/ci/helpers/runner.rb @@ -55,7 +55,7 @@ module API def current_runner_manager strong_memoize(:current_runner_manager) do system_xid = params.fetch(:system_id, LEGACY_SYSTEM_XID) - current_runner&.ensure_manager(system_xid) { |m| m.contacted_at = Time.current } + current_runner&.ensure_manager(system_xid) end end diff --git a/lib/api/commits.rb b/lib/api/commits.rb index c0222539c98..021b3a9437c 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -4,6 +4,7 @@ require 'mime/types' module API class Commits < ::API::Base include PaginationParams + include Helpers::Unidiff feature_category :source_code_management @@ -274,6 +275,7 @@ module API params do requires :sha, type: String, desc: 'A commit sha, or the name of a branch or tag' use :pagination + use :with_unidiff end get ':id/repository/commits/:sha/diff', requirements: API::COMMIT_ENDPOINT_REQUIREMENTS, urgency: :low do commit = user_project.commit(params[:sha]) @@ -282,7 +284,7 @@ module API raw_diffs = ::Kaminari.paginate_array(commit.diffs(expanded: true).diffs.to_a) - present paginate(raw_diffs), with: Entities::Diff + present paginate(raw_diffs), with: Entities::Diff, enable_unidiff: declared_params[:unidiff] end desc "Get a commit's comments" do diff --git a/lib/api/composer_packages.rb b/lib/api/composer_packages.rb index 56fa10dd7d4..7301afd7f4c 100644 --- a/lib/api/composer_packages.rb +++ b/lib/api/composer_packages.rb @@ -35,7 +35,7 @@ module API helpers do def packages strong_memoize(:packages) do - packages = ::Packages::Composer::PackagesFinder.new(current_user, user_group).execute + packages = ::Packages::Composer::PackagesFinder.new(current_user, find_authorized_group!).execute if params[:package_name].present? params[:package_name], params[:sha] = params[:package_name].split('$') @@ -52,7 +52,7 @@ module API end def presenter - @presenter ||= ::Packages::Composer::PackagesPresenter.new(user_group, packages, composer_v2?) + @presenter ||= ::Packages::Composer::PackagesPresenter.new(find_authorized_group!, packages, composer_v2?) end end @@ -66,7 +66,7 @@ module API resource :group, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do after_validation do - user_group + find_authorized_group! end desc 'Composer packages endpoint at group level' do @@ -78,7 +78,7 @@ module API ] tags %w[composer_packages] end - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true, deploy_token_allowed: true get ':id/-/packages/composer/packages', urgency: :low do presenter.root end @@ -95,7 +95,7 @@ module API params do requires :sha, type: String, desc: 'Shasum of current json', documentation: { example: '673594f85a55fe3c0eb45df7bd2fa9d95a1601ab' } end - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true, deploy_token_allowed: true get ':id/-/packages/composer/p/:sha', urgency: :low do presenter.provider end @@ -112,7 +112,7 @@ module API params do requires :package_name, type: String, file_path: true, desc: 'The Composer package name', documentation: { example: 'my-composer-package' } end - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true, deploy_token_allowed: true get ':id/-/packages/composer/p2/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true, urgency: :low do not_found! if packages.empty? @@ -131,7 +131,7 @@ module API params do requires :package_name, type: String, file_path: true, desc: 'The Composer package name', documentation: { example: 'my-composer-package' } end - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true, deploy_token_allowed: true get ':id/-/packages/composer/*package_name', requirements: COMPOSER_ENDPOINT_REQUIREMENTS, file_path: true, urgency: :low do not_found! if packages.empty? not_found! if params[:sha].blank? @@ -198,7 +198,7 @@ module API requires :sha, type: String, desc: 'Shasum of current json', documentation: { example: '673594f85a55fe3c0eb45df7bd2fa9d95a1601ab' } requires :package_name, type: String, file_path: true, desc: 'The Composer package name', documentation: { example: 'my-composer-package' } end - route_setting :authentication, job_token_allowed: true, basic_auth_personal_access_token: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: :basic_auth, basic_auth_personal_access_token: true, deploy_token_allowed: true get 'archives/*package_name', urgency: :default do project = authorized_user_project(action: :read_package) diff --git a/lib/api/concerns/packages/npm_endpoints.rb b/lib/api/concerns/packages/npm_endpoints.rb index 4278510e999..bfaba5c4d7a 100644 --- a/lib/api/concerns/packages/npm_endpoints.rb +++ b/lib/api/concerns/packages/npm_endpoints.rb @@ -76,12 +76,14 @@ module API ] failure [ { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, { code: 403, message: 'Forbidden' }, { code: 404, message: 'Not Found' } ] tags %w[npm_packages] end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, + authenticate_non_public: true get 'dist-tags', format: false, requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do package_name = params[:package_name] @@ -186,6 +188,7 @@ module API ] failure [ { code: 400, message: 'Bad Request' }, + { code: 401, message: 'Unauthorized' }, { code: 403, message: 'Forbidden' }, { code: 404, message: 'Not Found' } ] @@ -194,7 +197,8 @@ module API params do use :package_name end - route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true + route_setting :authentication, job_token_allowed: true, deploy_token_allowed: true, + authenticate_non_public: true get '*package_name', format: false, requirements: ::API::Helpers::Packages::Npm::NPM_ENDPOINT_REQUIREMENTS do package_name = params[:package_name] available_packages = @@ -224,9 +228,7 @@ module API ).execute if available_packages.any? && available_packages_to_user.empty? - forbidden! if current_user - - not_found!('Packages') + current_user ? forbidden! : unauthorized! end available_packages = available_packages_to_user diff --git a/lib/api/concerns/packages/nuget/private_endpoints.rb b/lib/api/concerns/packages/nuget/private_endpoints.rb index a166a7294f4..3a6261160e4 100644 --- a/lib/api/concerns/packages/nuget/private_endpoints.rb +++ b/lib/api/concerns/packages/nuget/private_endpoints.rb @@ -20,41 +20,6 @@ module API NON_NEGATIVE_INTEGER_REGEX = %r{\A(0|[1-9]\d*)\z} included do - helpers do - def find_packages(package_name) - packages = package_finder(package_name).execute - - not_found!('Packages') unless packages.exists? - - packages - end - - def find_package(package_name, package_version) - package = package_finder(package_name, package_version).execute - .first - - not_found!('Package') unless package - - package - end - - def package_finder(package_name, package_version = nil) - ::Packages::Nuget::PackageFinder.new( - current_user, - project_or_group, - package_name: package_name, - package_version: package_version, - client_version: headers['X-Nuget-Client-Version'] - ) - end - - def search_packages(_search_term, search_options) - ::Packages::Nuget::SearchService - .new(current_user, project_or_group, params[:q], search_options) - .execute - end - end - # https://docs.microsoft.com/en-us/nuget/api/registration-base-url-resource params do requires :package_name, type: String, desc: 'The NuGet package name', diff --git a/lib/api/debian_group_packages.rb b/lib/api/debian_group_packages.rb index 7c64dc2f877..9ceccbb5635 100644 --- a/lib/api/debian_group_packages.rb +++ b/lib/api/debian_group_packages.rb @@ -3,7 +3,7 @@ module API class DebianGroupPackages < ::API::Base PACKAGE_FILE_REQUIREMENTS = ::API::DebianProjectPackages::PACKAGE_FILE_REQUIREMENTS.merge( - project_id: %r{[0-9]+}.freeze + project_id: %r{[0-9]+} ).freeze resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do diff --git a/lib/api/entities/basic_project_details.rb b/lib/api/entities/basic_project_details.rb index f89e5adca6d..fa247370606 100644 --- a/lib/api/entities/basic_project_details.rb +++ b/lib/api/entities/basic_project_details.rb @@ -41,6 +41,10 @@ module API expose :namespace, using: 'API::Entities::NamespaceBasic' expose :custom_attributes, using: 'API::Entities::CustomAttribute', if: :with_custom_attributes + expose :repository_storage, documentation: { type: 'string', example: 'default' }, if: ->(project, options) { + Ability.allowed?(options[:current_user], :change_repository_storage, project) + } + # rubocop: disable CodeReuse/ActiveRecord def self.preload_relation(projects_relation, options = {}) # Preloading topics, should be done with using only `:topics`, diff --git a/lib/api/entities/bulk_import.rb b/lib/api/entities/bulk_import.rb index 75989cb4180..18f71048595 100644 --- a/lib/api/entities/bulk_import.rb +++ b/lib/api/entities/bulk_import.rb @@ -10,6 +10,7 @@ module API expose :source_type, documentation: { type: 'string', example: 'gitlab' } expose :created_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } expose :updated_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } + expose :has_failures, documentation: { type: 'boolean', example: false } end end end diff --git a/lib/api/entities/bulk_imports/entity.rb b/lib/api/entities/bulk_imports/entity.rb index 176d10b2580..7e9b9973e15 100644 --- a/lib/api/entities/bulk_imports/entity.rb +++ b/lib/api/entities/bulk_imports/entity.rb @@ -24,6 +24,7 @@ module API expose :updated_at, documentation: { type: 'dateTime', example: '2012-05-28T04:42:42-07:00' } expose :failures, using: EntityFailure, documentation: { is_array: true } expose :migrate_projects, documentation: { type: 'boolean', example: true } + expose :has_failures, documentation: { type: 'boolean', example: false } end end end diff --git a/lib/api/entities/diff.rb b/lib/api/entities/diff.rb index b9538893d32..cc53736a5b1 100644 --- a/lib/api/entities/diff.rb +++ b/lib/api/entities/diff.rb @@ -3,10 +3,12 @@ module API module Entities class Diff < Grape::Entity - expose :json_safe_diff, as: :diff, documentation: { + expose :diff, documentation: { type: 'string', example: '@@ -71,6 +71,8 @@\n...' - } + } do |instance, options| + options[:enable_unidiff] == true ? instance.unidiff : instance.json_safe_diff + end expose :new_path, documentation: { type: 'string', example: 'doc/update/5.4-to-6.0.md' } expose :old_path, documentation: { type: 'string', example: 'doc/update/5.4-to-6.0.md' } expose :a_mode, documentation: { type: 'string', example: '100755' } diff --git a/lib/api/entities/namespace.rb b/lib/api/entities/namespace.rb index 5e0630e0f7f..012dc467a1c 100644 --- a/lib/api/entities/namespace.rb +++ b/lib/api/entities/namespace.rb @@ -11,11 +11,15 @@ module API namespace.kind == 'group' && Ability.allowed?(opts[:current_user], :admin_group, namespace) end - expose :root_repository_size, documentation: { type: 'integer', example: 123 }, if: -> (namespace, opts) { expose_root_repository_size?(namespace, opts) } do |namespace, _| + expose :root_repository_size, documentation: { type: 'integer', example: 123 }, if: -> (namespace, opts) { admin_request_for_group?(namespace, opts) } do |namespace, _| namespace.root_storage_statistics&.repository_size end - def expose_root_repository_size?(namespace, opts) + expose :projects_count, documentation: { type: 'integer', example: 123 }, if: -> (namespace, opts) { admin_request_for_group?(namespace, opts) } do |namespace, _| + namespace.all_projects.count + end + + def admin_request_for_group?(namespace, opts) namespace.kind == 'group' && Ability.allowed?(opts[:current_user], :admin_group, namespace) end end diff --git a/lib/api/entities/namespace_basic.rb b/lib/api/entities/namespace_basic.rb index 4264326cdc2..ccc472e7d51 100644 --- a/lib/api/entities/namespace_basic.rb +++ b/lib/api/entities/namespace_basic.rb @@ -13,7 +13,7 @@ module API expose :web_url, documentation: { type: 'string', example: 'https://example.com/group/my_project' } do |namespace| if namespace.user_namespace? - Gitlab::Routing.url_helpers.user_url(namespace.owner) + Gitlab::Routing.url_helpers.user_url(namespace.owner || namespace.route.path) else namespace.web_url end diff --git a/lib/api/entities/project.rb b/lib/api/entities/project.rb index 0f947c85633..12e022bfb20 100644 --- a/lib/api/entities/project.rb +++ b/lib/api/entities/project.rb @@ -84,6 +84,7 @@ module API expose(:feature_flags_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :feature_flags) } expose(:infrastructure_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :infrastructure) } expose(:monitor_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :monitor) } + expose(:model_experiments_access_level, documentation: { type: 'string', example: 'enabled' }) { |project, options| project_feature_string_access_level(project, :model_experiments) } expose(:emails_disabled, documentation: { type: 'boolean' }) { |project, options| project.emails_disabled? } expose :emails_enabled, documentation: { type: 'boolean' } @@ -159,9 +160,6 @@ module API } expose :autoclose_referenced_issues, documentation: { type: 'boolean' } - expose :repository_storage, documentation: { type: 'string', example: 'default' }, if: ->(project, options) { - Ability.allowed?(options[:current_user], :change_repository_storage, project) - } # rubocop: disable CodeReuse/ActiveRecord def self.preload_resource(project) diff --git a/lib/api/entities/user_basic.rb b/lib/api/entities/user_basic.rb index 32e066b9f7e..9b1814251fe 100644 --- a/lib/api/entities/user_basic.rb +++ b/lib/api/entities/user_basic.rb @@ -4,6 +4,7 @@ module API module Entities class UserBasic < UserSafe expose :state, documentation: { type: 'string', example: 'active' } + expose :access_locked?, as: :locked, documentation: { type: 'boolean' } expose :avatar_url, documentation: { type: 'string', example: 'https://gravatar.com/avatar/1' } do |user, options| user.avatar_url(only_path: false) diff --git a/lib/api/entities/wiki_page.rb b/lib/api/entities/wiki_page.rb index 9d2a031cee8..0f3fdd586a3 100644 --- a/lib/api/entities/wiki_page.rb +++ b/lib/api/entities/wiki_page.rb @@ -15,7 +15,7 @@ module API current_user: options[:current_user] ) else - wiki_page.content + wiki_page.raw_content end end diff --git a/lib/api/go_proxy.rb b/lib/api/go_proxy.rb index 8fde40a4713..3933e07d150 100755 --- a/lib/api/go_proxy.rb +++ b/lib/api/go_proxy.rb @@ -10,7 +10,7 @@ module API urgency :low # basic semver, except case encoded (A => !a) - MODULE_VERSION_REGEX = /v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([-.!a-z0-9]+))?(?:\+([-.!a-z0-9]+))?/.freeze + MODULE_VERSION_REGEX = /v(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-([-.!a-z0-9]+))?(?:\+([-.!a-z0-9]+))?/ MODULE_VERSION_REQUIREMENTS = { module_version: MODULE_VERSION_REGEX }.freeze diff --git a/lib/api/group_export.rb b/lib/api/group_export.rb index 4cac707ff66..819cc4652f6 100644 --- a/lib/api/group_export.rb +++ b/lib/api/group_export.rb @@ -66,7 +66,8 @@ module API resource do before do - not_found! unless Gitlab::CurrentSettings.bulk_import_enabled? + not_found! unless Gitlab::CurrentSettings.bulk_import_enabled? || + Feature.enabled?(:override_bulk_import_disabled, current_user, type: :ops) end desc 'Start relations export' do diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index e967b88e500..56b157f662a 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -19,7 +19,7 @@ module API API_TOKEN_ENV = 'gitlab.api.token' API_EXCEPTION_ENV = 'gitlab.api.exception' API_RESPONSE_STATUS_CODE = 'gitlab.api.response_status_code' - INTEGER_ID_REGEX = /^-?\d+$/.freeze + INTEGER_ID_REGEX = /^-?\d+$/ def logger API.logger @@ -237,7 +237,7 @@ module API end def check_namespace_access(namespace) - return namespace if can?(current_user, :read_namespace, namespace) + return namespace if can?(current_user, :read_namespace_via_membership, namespace) not_found!('Namespace') end @@ -412,7 +412,7 @@ module API end def require_pages_enabled! - not_found! unless user_project.pages_available? + not_found! unless ::Gitlab::Pages.enabled? end def require_pages_config_enabled! @@ -462,8 +462,8 @@ module API items.search(text) end - def order_options_with_tie_breaker - order_by = if params[:order_by] == 'created_at' + def order_options_with_tie_breaker(override_created_at: true) + order_by = if params[:order_by] == 'created_at' && override_created_at 'id' else params[:order_by] @@ -700,14 +700,18 @@ module API Gitlab::AppLogger.warn("Redis tracking event failed for event: #{event_name}, message: #{error.message}") end - def track_event(event_name, user_id:, namespace_id: nil, project_id: nil) - return unless user_id.present? + def track_event(event_name, user:, send_snowplow_event: true, namespace_id: nil, project_id: nil) + return unless user.present? + + namespace = Namespace.find(namespace_id) if namespace_id + project = Project.find(project_id) if project_id Gitlab::InternalEvents.track_event( event_name, - user_id: user_id, - namespace_id: namespace_id, - project_id: project_id + send_snowplow_event: send_snowplow_event, + user: user, + namespace: namespace, + project: project ) rescue StandardError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_name: event_name) diff --git a/lib/api/helpers/import_github_helpers.rb b/lib/api/helpers/import_github_helpers.rb new file mode 100644 index 00000000000..1634e064d73 --- /dev/null +++ b/lib/api/helpers/import_github_helpers.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module API + module Helpers + module ImportGithubHelpers + def client + @client ||= Gitlab::GithubImport::Client.new(params[:personal_access_token], host: params[:github_hostname]) + end + + def access_params + { + github_access_token: params[:personal_access_token], + additional_access_tokens: params[:additional_access_tokens] + } + end + + def provider + :github + end + + def provider_unauthorized + error!("Access denied to your #{Gitlab::ImportSources.title(provider.to_s)} account.", 401) + end + + def too_many_requests + error!('Too Many Requests', 429) + end + end + end +end diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index 8f846fe7348..a08337a86ac 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -698,6 +698,12 @@ module API ], 'prometheus' => [ { + required: false, + name: :manual_configuration, + type: ::Grape::API::Boolean, + desc: 'When enabled, the default settings will be overridden with your custom configuration' + }, + { required: true, name: :api_url, type: String, diff --git a/lib/api/helpers/members_helpers.rb b/lib/api/helpers/members_helpers.rb index a82aed507fd..1a23dcd0d3c 100644 --- a/lib/api/helpers/members_helpers.rb +++ b/lib/api/helpers/members_helpers.rb @@ -22,6 +22,14 @@ module API authorize! :"read_#{source_type}_member", source end + def authorize_admin_source_member!(source_type, source) + authorize! :"admin_#{source_type}_member", source + end + + def authorize_update_source_member!(source_type, member) + authorize! :"update_#{source_type}_member", member + end + def authorize_admin_source!(source_type, source) authorize! :"admin_#{source_type}", source end diff --git a/lib/api/helpers/packages/maven.rb b/lib/api/helpers/packages/maven.rb index 71d1ba486ed..6c50f4c00a1 100644 --- a/lib/api/helpers/packages/maven.rb +++ b/lib/api/helpers/packages/maven.rb @@ -9,10 +9,12 @@ module API params :path_and_file_name do requires :path, type: String, + file_path: true, desc: 'Package path', documentation: { example: 'foo/bar/mypkg/1.0-SNAPSHOT' } requires :file_name, type: String, + file_path: true, desc: 'Package file name', documentation: { example: 'mypkg-1.0-SNAPSHOT.jar' } end @@ -38,7 +40,7 @@ module API project || group, path: params[:path], order_by_package_file: order_by_package_file - ).execute + ).execute&.last end def project diff --git a/lib/api/helpers/packages/npm.rb b/lib/api/helpers/packages/npm.rb index a80122c5309..ef3da055b19 100644 --- a/lib/api/helpers/packages/npm.rb +++ b/lib/api/helpers/packages/npm.rb @@ -102,8 +102,7 @@ module API def group group = find_group(params[:id]) - not_found!('Group') unless can?(current_user, :read_group, group) - group + check_group_access(group) end strong_memoize_attr :group diff --git a/lib/api/helpers/packages/nuget.rb b/lib/api/helpers/packages/nuget.rb new file mode 100644 index 00000000000..19192b31b16 --- /dev/null +++ b/lib/api/helpers/packages/nuget.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module API + module Helpers + module Packages + module Nuget + def find_packages(package_name) + packages = package_finder(package_name).execute + + not_found!('Packages') unless packages.exists? + + packages + end + + def find_package(package_name, package_version) + package = package_finder(package_name, package_version).execute.first + + not_found!('Package') unless package + + package + end + + def package_finder(package_name, package_version = nil) + ::Packages::Nuget::PackageFinder.new( + current_user, + project_or_group, + package_name: package_name, + package_version: package_version, + client_version: headers['X-Nuget-Client-Version'] + ) + end + + def search_packages(_search_term, search_options) + ::Packages::Nuget::SearchService + .new(current_user, project_or_group, params[:q], search_options) + .execute + end + end + end + end +end diff --git a/lib/api/helpers/projects_helpers.rb b/lib/api/helpers/projects_helpers.rb index 8a0ec1c1abf..23e83d9d54f 100644 --- a/lib/api/helpers/projects_helpers.rb +++ b/lib/api/helpers/projects_helpers.rb @@ -39,6 +39,7 @@ module API optional :feature_flags_access_level, type: String, values: %w[disabled private enabled], desc: 'Feature flags access level. One of `disabled`, `private` or `enabled`' optional :infrastructure_access_level, type: String, values: %w[disabled private enabled], desc: 'Infrastructure access level. One of `disabled`, `private` or `enabled`' optional :monitor_access_level, type: String, values: %w[disabled private enabled], desc: 'Monitor access level. One of `disabled`, `private` or `enabled`' + optional :model_experiments_access_level, type: String, values: %w[disabled private enabled], desc: 'Model experiments access level. One of `disabled`, `private` or `enabled`' optional :emails_disabled, type: Boolean, desc: 'Deprecated: Use emails_enabled instead.' optional :emails_enabled, type: Boolean, desc: 'Enable email notifications' @@ -195,6 +196,7 @@ module API :feature_flags_access_level, :infrastructure_access_level, :monitor_access_level, + :model_experiments_access_level, # TODO: remove in API v5, replaced by *_access_level :issues_enabled, diff --git a/lib/api/helpers/unidiff.rb b/lib/api/helpers/unidiff.rb new file mode 100644 index 00000000000..aabc0acd454 --- /dev/null +++ b/lib/api/helpers/unidiff.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module API + module Helpers + module Unidiff + extend ActiveSupport::Concern + + included do + helpers do + params :with_unidiff do + optional :unidiff, type: ::Grape::API::Boolean, default: false, desc: 'A diff in a Unified diff format' + end + end + end + end + end +end diff --git a/lib/api/import_bitbucket_server.rb b/lib/api/import_bitbucket_server.rb index f315ae5afff..1635e5ab07b 100644 --- a/lib/api/import_bitbucket_server.rb +++ b/lib/api/import_bitbucket_server.rb @@ -40,6 +40,8 @@ module API requires :bitbucket_server_repo, type: String, desc: 'BitBucket Server Repository Name' optional :new_name, type: String, desc: 'New repo name' optional :new_namespace, type: String, desc: 'Namespace to import repo into' + optional :timeout_strategy, type: String, values: ::ProjectImportData::TIMEOUT_STRATEGIES, + desc: 'Strategy for behavior on timeouts' end post 'import/bitbucket_server' do diff --git a/lib/api/import_github.rb b/lib/api/import_github.rb index ab7ac6624a8..29dfa7c9f29 100644 --- a/lib/api/import_github.rb +++ b/lib/api/import_github.rb @@ -10,38 +10,7 @@ module API rescue_from Octokit::Unauthorized, with: :provider_unauthorized rescue_from Gitlab::GithubImport::RateLimitError, with: :too_many_requests - helpers do - def client - @client ||= if Feature.enabled?(:remove_legacy_github_client) - Gitlab::GithubImport::Client.new(params[:personal_access_token], host: params[:github_hostname]) - else - Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], **client_options) - end - end - - def access_params - { - github_access_token: params[:personal_access_token], - additional_access_tokens: params[:additional_access_tokens] - } - end - - def client_options - { host: params[:github_hostname] } - end - - def provider - :github - end - - def provider_unauthorized - error!("Access denied to your #{Gitlab::ImportSources.title(provider.to_s)} account.", 401) - end - - def too_many_requests - error!('Too Many Requests', 429) - end - end + helpers ::API::Helpers::ImportGithubHelpers desc 'Import a GitHub project' do detail 'This feature was introduced in GitLab 11.3.4.' @@ -62,6 +31,8 @@ module API requires :target_namespace, type: String, allow_blank: false, desc: 'Namespace or group to import repository into' optional :github_hostname, type: String, desc: 'Custom GitHub enterprise hostname' optional :optional_stages, type: Hash, desc: 'Optional stages of import to be performed' + optional :timeout_strategy, type: String, values: ::ProjectImportData::TIMEOUT_STRATEGIES, + desc: 'Strategy for behavior on timeouts' optional :additional_access_tokens, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, diff --git a/lib/api/internal/kubernetes.rb b/lib/api/internal/kubernetes.rb index a88c8b69b81..b8a2fde4e36 100644 --- a/lib/api/internal/kubernetes.rb +++ b/lib/api/internal/kubernetes.rb @@ -97,9 +97,7 @@ module API user = if params[:access_type] == 'session_cookie' retrieve_user_from_session_cookie elsif params[:access_type] == 'personal_access_token' - u = retrieve_user_from_personal_access_token - bad_request!('PAT authentication is not enabled') unless Feature.enabled?(:k8s_proxy_pat, u) - u + retrieve_user_from_personal_access_token end bad_request!('Unable to get user from request data') if user.nil? diff --git a/lib/api/invitations.rb b/lib/api/invitations.rb index 828f4b419ef..34f9538b047 100644 --- a/lib/api/invitations.rb +++ b/lib/api/invitations.rb @@ -26,8 +26,6 @@ module API optional :user_id, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, desc: 'The user ID of the new member or multiple IDs separated by commas.' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'invitations-api' - optional :tasks_to_be_done, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Tasks the inviter wants the member to do' - optional :tasks_project_id, type: Integer, desc: 'The project ID in which to create the task issues' end post ":id/invitations", urgency: :low do ::Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/354016') @@ -35,7 +33,12 @@ module API bad_request!('Must provide either email or user_id as a parameter') if params[:email].blank? && params[:user_id].blank? source = find_source(source_type, params[:id]) - authorize_admin_source!(source_type, source) + + if ::Feature.enabled?(:admin_group_member, source) + authorize_admin_source_member!(source_type, source) + else + authorize_admin_source!(source_type, source) + end create_service_params = params.merge(source: source) @@ -58,7 +61,11 @@ module API source = find_source(source_type, params[:id]) query = params[:query] - authorize_admin_source!(source_type, source) + if ::Feature.enabled?(:admin_group_member, source) + authorize_admin_source_member!(source_type, source) + else + authorize_admin_source!(source_type, source) + end invitations = paginate(retrieve_member_invitations(source, query)) @@ -77,7 +84,12 @@ module API put ":id/invitations/:email", requirements: { email: %r{[^/]+} } do source = find_source(source_type, params.delete(:id)) invite_email = params[:email] - authorize_admin_source!(source_type, source) + + if ::Feature.enabled?(:admin_group_member, source) + authorize_admin_source_member!(source_type, source) + else + authorize_admin_source!(source_type, source) + end invite = retrieve_member_invitations(source, invite_email).first not_found! unless invite @@ -114,7 +126,12 @@ module API delete ":id/invitations/:email", requirements: { email: %r{[^/]+} } do source = find_source(source_type, params[:id]) invite_email = params[:email] - authorize_admin_source!(source_type, source) + + if ::Feature.enabled?(:admin_group_member, source) + authorize_admin_source_member!(source_type, source) + else + authorize_admin_source!(source_type, source) + end invite = retrieve_member_invitations(source, invite_email).first not_found! unless invite diff --git a/lib/api/lint.rb b/lib/api/lint.rb index 71965fc05c9..26619e6924f 100644 --- a/lib/api/lint.rb +++ b/lib/api/lint.rb @@ -6,12 +6,16 @@ module API resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do desc 'Validates a CI YAML configuration with a namespace' do - detail 'Checks if a project’s latest (HEAD of the project’s default branch) .gitlab-ci.yml configuration is - valid' + detail 'Checks if a project’s .gitlab-ci.yml configuration in a given commit (by default HEAD of the + project’s default branch) is valid' success Entities::Ci::Lint::Result tags %w[ci_lint] + failure [ + { code: 404, message: 'Not found' } + ] end params do + optional :sha, type: String, desc: 'The commit hash or name of a repository branch or tag. Defaults to the HEAD of the project’s default branch' optional :dry_run, type: Boolean, default: false, desc: 'Run pipeline creation simulation, or only do static check. This is false by default' optional :include_jobs, type: Boolean, desc: 'If the list of jobs that would exist in a static check or pipeline simulation should be included in the response. This is false by default' @@ -21,12 +25,13 @@ module API get ':id/ci/lint', urgency: :low do authorize_read_code! - if user_project.commit.present? - content = user_project.repository.gitlab_ci_yml_for(user_project.commit.id, user_project.ci_config_path_or_default) - end + sha = params[:sha] || user_project.repository.root_ref_sha + not_found! 'Commit' unless user_project.commit(sha).present? + + content = user_project.repository.gitlab_ci_yml_for(sha, user_project.ci_config_path_or_default) result = Gitlab::Ci::Lint - .new(project: user_project, current_user: current_user) + .new(project: user_project, current_user: current_user, sha: sha) .validate(content, dry_run: params[:dry_run], ref: params[:ref] || user_project.default_branch) present result, with: Entities::Ci::Lint::Result, current_user: current_user, include_jobs: params[:include_jobs] diff --git a/lib/api/members.rb b/lib/api/members.rb index 337706f36e1..bdbdea70da0 100644 --- a/lib/api/members.rb +++ b/lib/api/members.rb @@ -114,13 +114,15 @@ module API requires :user_id, types: [Integer, String], desc: 'The user ID of the new member or multiple IDs separated by commas.' optional :expires_at, type: DateTime, desc: 'Date string in the format YEAR-MONTH-DAY' optional :invite_source, type: String, desc: 'Source that triggered the member creation process', default: 'members-api' - optional :tasks_to_be_done, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Tasks the inviter wants the member to do' - optional :tasks_project_id, type: Integer, desc: 'The project ID in which to create the task issues' end post ":id/members", feature_category: feature_category do source = find_source(source_type, params[:id]) - authorize_admin_source!(source_type, source) + if ::Feature.enabled?(:admin_group_member, source) + authorize_admin_source_member!(source_type, source) + else + authorize_admin_source!(source_type, source) + end create_service_params = params.merge(source: source) @@ -144,10 +146,14 @@ module API # rubocop: disable CodeReuse/ActiveRecord put ":id/members/:user_id", feature_category: feature_category do source = find_source(source_type, params.delete(:id)) - authorize_admin_source!(source_type, source) - member = source_members(source).find_by!(user_id: params[:user_id]) + if ::Feature.enabled?(:admin_group_member, source) + authorize_update_source_member!(source_type, member) + else + authorize_admin_source!(source_type, source) + end + result = ::Members::UpdateService .new(current_user, declared_params(include_missing: false)) .execute(member) diff --git a/lib/api/merge_request_diffs.rb b/lib/api/merge_request_diffs.rb index e7193035ce0..9b8468b6efb 100644 --- a/lib/api/merge_request_diffs.rb +++ b/lib/api/merge_request_diffs.rb @@ -4,6 +4,7 @@ module API # MergeRequestDiff API class MergeRequestDiffs < ::API::Base include PaginationParams + include Helpers::Unidiff before { authenticate! } @@ -39,12 +40,13 @@ module API params do requires :merge_request_iid, type: Integer, desc: 'The internal ID of the merge request' requires :version_id, type: Integer, desc: 'The ID of the merge request diff version' + use :with_unidiff end get ":id/merge_requests/:merge_request_iid/versions/:version_id", urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present_cached merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull, cache_context: nil + present_cached merge_request.merge_request_diffs.find(params[:version_id]), with: Entities::MergeRequestDiffFull, cache_context: nil, enable_unidiff: declared_params[:unidiff] end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index 1c0b9c56aa7..b8285bbd109 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -3,6 +3,7 @@ module API class MergeRequests < ::API::Base include PaginationParams + include Helpers::Unidiff CONTEXT_COMMITS_POST_LIMIT = 20 @@ -30,8 +31,15 @@ module API params :optional_params_ee do end + params :optional_merge_params_ee do + end + params :optional_merge_requests_search_params do end + + def ci_params + {} + end end def self.update_params_at_least_one_of @@ -68,7 +76,7 @@ module API args[:scope] = args[:scope].underscore if args[:scope] merge_requests = MergeRequestsFinder.new(current_user, args).execute - .reorder(order_options_with_tie_breaker) + .reorder(order_options_with_tie_breaker(override_created_at: false)) merge_requests = paginate(merge_requests) .preload(:source_project, :target_project) @@ -231,6 +239,10 @@ module API use :optional_params_ee end + + params :optional_merge_params do + use :optional_merge_params_ee + end end desc 'List project merge requests' do @@ -505,6 +517,9 @@ module API ] tags %w[merge_requests] end + params do + use :with_unidiff + end get ':id/merge_requests/:merge_request_iid/changes', feature_category: :code_review_workflow, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) @@ -512,7 +527,8 @@ module API with: Entities::MergeRequestChanges, current_user: current_user, project: user_project, - access_raw_diffs: to_boolean(params.fetch(:access_raw_diffs, false)) + access_raw_diffs: to_boolean(params.fetch(:access_raw_diffs, false)), + enable_unidiff: declared_params[:unidiff] end desc 'Get the merge request diffs' do @@ -526,11 +542,12 @@ module API end params do use :pagination + use :with_unidiff end get ':id/merge_requests/:merge_request_iid/diffs', feature_category: :code_review_workflow, urgency: :low do merge_request = find_merge_request_with_access(params[:merge_request_iid]) - present paginate(merge_request.merge_request_diff.paginated_diffs(params[:page], params[:per_page])).diffs, with: Entities::Diff + present paginate(merge_request.merge_request_diff.paginated_diffs(params[:page], params[:per_page])).diffs, with: Entities::Diff, enable_unidiff: declared_params[:unidiff] end desc 'Get single merge request pipelines' do @@ -636,6 +653,8 @@ module API desc: 'If `true`, the merge request is merged when the pipeline succeeds.' optional :sha, type: String, desc: 'If present, then this SHA must match the HEAD of the source branch, otherwise the merge fails.' optional :squash, type: Grape::API::Boolean, desc: 'If `true`, the commits are squashed into a single commit on merge.' + + use :optional_merge_params end put ':id/merge_requests/:merge_request_iid/merge', feature_category: :code_review_workflow, urgency: :low do Gitlab::QueryLimiting.disable!('https://gitlab.com/gitlab-org/gitlab/-/issues/4796') @@ -664,7 +683,7 @@ module API squash_commit_message: params[:squash_commit_message], should_remove_source_branch: params[:should_remove_source_branch], sha: params[:sha] || merge_request.diff_head_sha - ).compact + ).merge(ci_params).compact if immediately_mergeable ::MergeRequests::MergeService diff --git a/lib/api/ml/mlflow/runs.rb b/lib/api/ml/mlflow/runs.rb index 5b6afffaae1..ac052d8bff5 100644 --- a/lib/api/ml/mlflow/runs.rb +++ b/lib/api/ml/mlflow/runs.rb @@ -65,7 +65,7 @@ module API type: String, desc: 'Token for pagination' end - get 'search', urgency: :low do + post 'search', urgency: :low do params[:experiment_id] = params[:experiment_ids][0] max_results = [params[:max_results], 1000].min diff --git a/lib/api/nuget_group_packages.rb b/lib/api/nuget_group_packages.rb index 229032f7a5a..7a6872ee82f 100644 --- a/lib/api/nuget_group_packages.rb +++ b/lib/api/nuget_group_packages.rb @@ -11,6 +11,7 @@ module API class NugetGroupPackages < ::API::Base helpers ::API::Helpers::PackagesHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers + helpers ::API::Helpers::Packages::Nuget include ::API::Helpers::Authentication feature_category :package_registry diff --git a/lib/api/nuget_project_packages.rb b/lib/api/nuget_project_packages.rb index dbc789c68b6..46b388a2fda 100644 --- a/lib/api/nuget_project_packages.rb +++ b/lib/api/nuget_project_packages.rb @@ -11,6 +11,7 @@ module API class NugetProjectPackages < ::API::Base helpers ::API::Helpers::PackagesHelpers helpers ::API::Helpers::Packages::BasicAuthHelpers + helpers ::API::Helpers::Packages::Nuget include ::API::Helpers::Authentication feature_category :package_registry @@ -113,9 +114,7 @@ module API track_package_event( symbol_package ? 'push_symbol_package' : 'push_package', :nuget, - **{ category: 'API::NugetPackages', - project: package.project, - namespace: package.project.namespace }.tap { |args| args[:feed] = 'v2' if request.path.include?('nuget/v2') } + **track_package_event_attrs(package.project) ) end rescue ObjectStorage::RemoteStoreError => e @@ -148,6 +147,16 @@ module API present odata_entry end + + def track_package_event_attrs(project) + attrs = { + category: 'API::NugetPackages', + project: project, + namespace: project.namespace + } + attrs[:feed] = 'v2' if request.path.include?('nuget/v2') + attrs + end end params do @@ -216,9 +225,7 @@ module API track_package_event( params[:format] == 'snupkg' ? 'pull_symbol_package' : 'pull_package', :nuget, - category: 'API::NugetPackages', - project: package_file.project, - namespace: package_file.project.namespace + **track_package_event_attrs(package.project) ) # nuget and dotnet don't support 302 Moved status codes, supports_direct_download has to be set to false @@ -227,7 +234,7 @@ module API end end - # To support an additional authentication option for publish endpoints, + # To support an additional authentication option for publish/delete endpoints, # we redefine the `authenticate_with` method by combining the previous # authentication option with the new one. authenticate_with do |accept| @@ -305,6 +312,31 @@ module API authorize_nuget_upload end + desc 'The NuGet Package Delete endpoint' do + detail 'This feature was introduced in GitLab 16.5' + success code: 204 + failure [ + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 404, message: 'Not Found' } + ] + tags %w[nuget_packages] + end + params do + requires :package_name, type: String, allow_blank: false, desc: 'The NuGet package name', regexp: Gitlab::Regex.nuget_package_name_regex, documentation: { example: 'mynugetpkg' } + requires :package_version, type: String, allow_blank: false, desc: 'The NuGet package version', regexp: Gitlab::Regex.nuget_version_regex, documentation: { example: '1.0.1' } + end + delete '*package_name/*package_version', format: false, urgency: :low do + authorize_destroy_package!(project_or_group) + + package = find_package(params[:package_name], params[:package_version]) + destroy_conditionally!(package) do |package| + ::Packages::MarkPackageForDestructionService.new(container: package, current_user: current_user).execute + + track_package_event('delete_package', :nuget, category: 'API::NugetPackages', project: package.project, namespace: package.project.namespace) + end + end + namespace '/v2' do desc 'The NuGet V2 Feed Package Publish endpoint' do detail 'This feature was introduced in GitLab 16.2' diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb index 7467b8e564e..25848d91550 100644 --- a/lib/api/project_export.rb +++ b/lib/api/project_export.rb @@ -110,7 +110,8 @@ module API resource do before do - not_found! unless Gitlab::CurrentSettings.bulk_import_enabled? + not_found! unless Gitlab::CurrentSettings.bulk_import_enabled? || + Feature.enabled?(:override_bulk_import_disabled, current_user, type: :ops) authorize_admin_project end diff --git a/lib/api/projects_relation_builder.rb b/lib/api/projects_relation_builder.rb index bb1420534f1..b7c3221d9a0 100644 --- a/lib/api/projects_relation_builder.rb +++ b/lib/api/projects_relation_builder.rb @@ -16,7 +16,7 @@ module API Preloaders::UserMaxAccessLevelInProjectsPreloader.new(projects_relation, options[:current_user]).execute if options[:current_user] - options[:current_user].preloaded_member_roles_for_projects(projects_relation) if options[:current_user] + preload_member_roles(projects_relation, options[:current_user]) if options[:current_user] Preloaders::SingleHierarchyProjectGroupPlansPreloader.new(projects_relation).execute if options[:single_hierarchy] preload_groups(projects_relation) if options[:with] == Entities::Project @@ -62,6 +62,12 @@ module API def projects_for_group_preload(projects_relation) projects_relation.select { |project| project.namespace.type == Group.sti_name } end + + def preload_member_roles(projects, user) + # overridden in EE + end end end end + +API::ProjectsRelationBuilder.prepend_mod diff --git a/lib/api/repositories.rb b/lib/api/repositories.rb index 98316bf1d4b..0f1426dde99 100644 --- a/lib/api/repositories.rb +++ b/lib/api/repositories.rb @@ -5,6 +5,7 @@ require 'mime/types' module API class Repositories < ::API::Base include PaginationParams + include Helpers::Unidiff content_type :txt, 'text/plain' @@ -202,6 +203,7 @@ module API documentation: { example: 'feature' } optional :from_project_id, type: Integer, desc: 'The project to compare from', documentation: { example: 1 } optional :straight, type: Boolean, desc: 'Comparison method, `true` for direct comparison between `from` and `to` (`from`..`to`), `false` to compare using merge base (`from`...`to`)', default: false + use :with_unidiff end get ':id/repository/compare', urgency: :low do target_project = fetch_target_project(current_user, user_project, params) @@ -220,7 +222,7 @@ module API compare = CompareService.new(user_project, params[:to]).execute(target_project, params[:from], straight: params[:straight]) if compare - present compare, with: Entities::Compare, current_user: current_user + present compare, with: Entities::Compare, current_user: current_user, enable_unidiff: declared_params[:unidiff] else not_found!("Ref") end diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 9616efbfe37..9120421fadf 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -224,6 +224,7 @@ module API requires :slack_app_verification_token, type: String, desc: 'The verification token of the GitLab for Slack app. This method of authentication is deprecated by Slack and used only for authenticating slash commands from the app' end optional :namespace_aggregation_schedule_lease_duration_in_seconds, type: Integer, desc: 'Maximum duration (in seconds) between refreshes of namespace statistics (Default: 300)' + optional :project_jobs_api_rate_limit, type: Integer, desc: 'Maximum authenticated requests to /project/:id/jobs per minute' Gitlab::SSHPublicKey.supported_types.each do |type| optional :"#{type}_key_restriction", diff --git a/lib/api/usage_data.rb b/lib/api/usage_data.rb index 0a343093c33..0d1c6cb2281 100644 --- a/lib/api/usage_data.rb +++ b/lib/api/usage_data.rb @@ -77,7 +77,8 @@ module API track_event( event_name, - user_id: current_user.id, + send_snowplow_event: false, + user: current_user, namespace_id: namespace_id, project_id: project_id ) diff --git a/lib/api/users.rb b/lib/api/users.rb index a01ace3a9c3..dd9cb2ee019 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -141,11 +141,7 @@ module API users = users.preload(:user_detail) - if Feature.enabled?(:api_keyset_pagination_multi_order) - present paginate_with_strategies(users), options - else - present paginate(users), options - end + present paginate_with_strategies(users), options end # rubocop: enable CodeReuse/ActiveRecord @@ -1373,6 +1369,33 @@ module API get 'status', feature_category: :user_profile do present current_user.status || {}, with: Entities::UserStatus end + + resource :personal_access_tokens do + desc 'Create a personal access token with limited scopes for the currently authenticated user' do + detail 'This feature was introduced in GitLab 16.5' + success Entities::PersonalAccessTokenWithToken + end + params do + requires :name, type: String, desc: 'The name of the personal access token' + # NOTE: for security reasons only the k8s_proxy scope is allowed at the moment. + # See details in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/131923#note_1571272897 + # and in https://gitlab.com/gitlab-org/gitlab/-/issues/425171 + requires :scopes, type: Array[String], coerce_with: ::API::Validations::Types::CommaSeparatedToArray.coerce, values: [::Gitlab::Auth::K8S_PROXY_SCOPE].map(&:to_s), + desc: 'The array of scopes of the personal access token' + optional :expires_at, type: Date, default: -> { 1.day.from_now.to_date }, desc: 'The expiration date in the format YEAR-MONTH-DAY of the personal access token' + end + post feature_category: :system_access do + response = ::PersonalAccessTokens::CreateService.new( + current_user: current_user, target_user: current_user, params: declared_params(include_missing: false) + ).execute + + if response.success? + present response.payload[:personal_access_token], with: Entities::PersonalAccessTokenWithToken + else + render_api_error!(response.message, response.http_status || :unprocessable_entity) + end + end + end end end end diff --git a/lib/api/v3/github.rb b/lib/api/v3/github.rb index 7348ed612fc..0ce5cdd06de 100644 --- a/lib/api/v3/github.rb +++ b/lib/api/v3/github.rb @@ -9,7 +9,7 @@ module API module V3 class Github < ::API::Base - NO_SLASH_URL_PART_REGEX = %r{[^/]+}.freeze + NO_SLASH_URL_PART_REGEX = %r{[^/]+} ENDPOINT_REQUIREMENTS = { namespace: NO_SLASH_URL_PART_REGEX, project: NO_SLASH_URL_PART_REGEX, diff --git a/lib/api/validations/validators/bulk_imports.rb b/lib/api/validations/validators/bulk_imports.rb index 77d76c98e00..b09df7b7133 100644 --- a/lib/api/validations/validators/bulk_imports.rb +++ b/lib/api/validations/validators/bulk_imports.rb @@ -6,26 +6,14 @@ module API module BulkImports class DestinationSlugPath < Grape::Validations::Validators::Base def validate_param!(attr_name, params) - if Feature.disabled?(:restrict_special_characters_in_namespace_path) - return if params[attr_name] =~ Gitlab::Regex.group_path_regex + return if params[attr_name] =~ Gitlab::Regex.oci_repository_path_regex - raise Grape::Exceptions::Validation.new( - params: [@scope.full_name(attr_name)], - message: "#{Gitlab::Regex.group_path_regex_message} " \ - "It can only contain alphanumeric characters, periods, underscores, and dashes. " \ - "For example, 'destination_namespace' not 'destination/namespace'" - ) - else - return if params[attr_name] =~ Gitlab::Regex.oci_repository_path_regex - - raise Grape::Exceptions::Validation.new( - params: [@scope.full_name(attr_name)], - message: "#{Gitlab::Regex.oci_repository_path_regex_message} " \ - "It can only contain alphanumeric characters, periods, underscores, and dashes. " \ - "For example, 'destination_namespace' not 'destination/namespace'" - ) - - end + raise Grape::Exceptions::Validation.new( + params: [@scope.full_name(attr_name)], + message: "#{Gitlab::Regex.oci_repository_path_regex_message} " \ + "It can only contain alphanumeric characters, periods, underscores, and dashes. " \ + "For example, 'destination_namespace' not 'destination/namespace'" + ) end end diff --git a/lib/api/validations/validators/git_ref.rb b/lib/api/validations/validators/git_ref.rb index 711c272ab4e..4e113a4ef67 100644 --- a/lib/api/validations/validators/git_ref.rb +++ b/lib/api/validations/validators/git_ref.rb @@ -10,7 +10,7 @@ module API # We have skipped some checks that are optional and can be skipped for exception. # We also check for control characters, More info on ctrl chars - https://ruby-doc.org/core-2.7.0/Regexp.html#class-Regexp-label-Character+Classes INVALID_CHARS = Regexp.union('..', '\\', '@', '@{', ' ', '~', '^', ':', '*', '?', '[', /[[:cntrl:]]/).freeze - GIT_REF_LENGTH = (1..1024).freeze + GIT_REF_LENGTH = (1..1024) def validate_param!(attr_name, params) revision = params[attr_name] diff --git a/lib/api/vs_code/settings/entities/vs_code_manifest.rb b/lib/api/vs_code/settings/entities/vs_code_manifest.rb new file mode 100644 index 00000000000..5c5cc5dd116 --- /dev/null +++ b/lib/api/vs_code/settings/entities/vs_code_manifest.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module API + module VsCode + module Settings + module Entities + class VsCodeManifest < Grape::Entity + expose :latest + expose :session, documentation: { type: 'string', example: '1' } + end + end + end + end +end diff --git a/lib/api/vs_code/settings/entities/vs_code_setting.rb b/lib/api/vs_code/settings/entities/vs_code_setting.rb new file mode 100644 index 00000000000..f166d2281fd --- /dev/null +++ b/lib/api/vs_code/settings/entities/vs_code_setting.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module API + module VsCode + module Settings + module Entities + class VsCodeSetting < Grape::Entity + expose :content, expose_nil: false + expose :machines, expose_nil: false + expose :version + expose :machine_id, as: :machineId, expose_nil: false + end + end + end + end +end diff --git a/lib/api/vs_code/settings/vs_code_settings_sync.rb b/lib/api/vs_code/settings/vs_code_settings_sync.rb new file mode 100644 index 00000000000..dc22496e380 --- /dev/null +++ b/lib/api/vs_code/settings/vs_code_settings_sync.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module API + module VsCode + module Settings + class VsCodeSettingsSync < ::API::Base + include ::VsCode::Settings + + feature_category :web_ide + + before do + authenticate! + + header 'Access-Control-Expose-Headers', 'etag' + end + + resource :vscode do + resource :settings_sync do + content_type :json, 'application/json' + content_type :json, 'text/plain' + + desc 'Get the settings manifest for Settings Sync' do + success [Entities::VsCodeManifest] + tags %w[vscode] + end + get '/v1/manifest' do + settings = SettingsFinder.new(current_user, SETTINGS_TYPES).execute + presenter = VsCodeManifestPresenter.new(settings) + + present presenter, with: Entities::VsCodeManifest + end + + desc 'Get a specific setting resource' do + success [Entities::VsCodeSetting] + tags %w[vscode] + end + params do + requires :resource_name, type: String, desc: 'Name of the resource such as settings' + requires :id, type: String, desc: 'ID of the resource to retrieve' + end + get '/v1/resource/:resource_name/:id' do + authenticate! + + setting_name = params[:resource_name] + setting = nil + + if params[:resource_name] == 'machines' + setting = DEFAULT_MACHINE + else + settings = SettingsFinder.new(current_user, [setting_name]).execute + setting = settings.first if settings.present? + end + + if setting.nil? + status :no_content + header :etag, NO_CONTENT_ETAG + body false + else + header :etag, setting[:uuid] + presenter = VsCodeSettingPresenter.new setting + present presenter, with: Entities::VsCodeSetting + end + end + + desc 'Update a specific setting' + params do + requires :resource_name, type: String, desc: 'Name of the resource such as settings' + end + post '/v1/resource/:resource_name' do + authenticate! + + response = CreateOrUpdateService.new(current_user: current_user, params: { + content: params[:content], + version: params[:version], + setting_type: params[:resource_name] + }).execute + + if response.success? + header 'Access-Control-Expose-Headers', 'etag' + header 'Etag', response.payload[:uuid] + present "OK" + else + error!(response.message, 400) + end + end + end + end + end + end + end +end diff --git a/lib/atlassian/jira_connect/jwt/asymmetric.rb b/lib/atlassian/jira_connect/jwt/asymmetric.rb index 8698be70eb9..470b1fc8c9b 100644 --- a/lib/atlassian/jira_connect/jwt/asymmetric.rb +++ b/lib/atlassian/jira_connect/jwt/asymmetric.rb @@ -14,7 +14,7 @@ module Atlassian ALGORITHM = 'RS256' DEFAULT_PUBLIC_KEY_CDN_URL = 'https://connect-install-keys.atlassian.com' PROXY_PUBLIC_KEY_PATH = '/-/jira_connect/public_keys' - UUID4_REGEX = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/.freeze + UUID4_REGEX = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/ def initialize(token, verification_claims) @token = token diff --git a/lib/atlassian/jira_connect/serializers/pull_request_entity.rb b/lib/atlassian/jira_connect/serializers/pull_request_entity.rb index e2dc197969b..437a2301450 100644 --- a/lib/atlassian/jira_connect/serializers/pull_request_entity.rb +++ b/lib/atlassian/jira_connect/serializers/pull_request_entity.rb @@ -20,6 +20,9 @@ module Atlassian end expose :title expose :author, using: JiraConnect::Serializers::AuthorEntity + expose :reviewers do |mr| + JiraConnect::Serializers::ReviewerEntity.represent(mr.merge_request_reviewers, merge_request: mr) + end expose :commentCount do |mr| if options[:user_notes_count] options[:user_notes_count].fetch(mr.id, 0) diff --git a/lib/atlassian/jira_connect/serializers/reviewer_entity.rb b/lib/atlassian/jira_connect/serializers/reviewer_entity.rb new file mode 100644 index 00000000000..8fc1acacadc --- /dev/null +++ b/lib/atlassian/jira_connect/serializers/reviewer_entity.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Atlassian + module JiraConnect + module Serializers + class ReviewerEntity < Grape::Entity + include Gitlab::Routing + + expose :name do |reviewer| + reviewer.reviewer.name + end + expose :email do |reviewer| + reviewer.reviewer.email + end + + expose :approvalStatus do |reviewer, options| + interaction = Users::MergeRequestInteraction.new( + user: reviewer.reviewer, merge_request: options[:merge_request] + ) + + if interaction.approved? + 'APPROVED' + elsif interaction.reviewed? + 'NEEDSWORK' + else + 'UNAPPROVED' + end + end + end + end + end +end diff --git a/lib/aws/s3_client.rb b/lib/aws/s3_client.rb new file mode 100644 index 00000000000..bdaea7efabf --- /dev/null +++ b/lib/aws/s3_client.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Aws + class S3Client + def initialize(access_key_id, secret_access_key, aws_region) + credentials = Aws::Credentials.new(access_key_id, secret_access_key) + @s3_client = Aws::S3::Client.new(region: aws_region, credentials: credentials) + end + + def upload_object(key, bucket, body, content_type = 'application/json') + @s3_client.put_object(key: key, bucket: bucket, body: body, content_type: content_type) + end + end +end diff --git a/lib/backup/database.rb b/lib/backup/database.rb index f70a7e41862..58a8c19c1ce 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -75,7 +75,7 @@ module Backup end override :restore - def restore(destination_dir) + def restore(destination_dir, backup_id) base_models_for_backup.each do |database_name, _base_model| backup_model = Backup::DatabaseModel.new(database_name) diff --git a/lib/backup/database_model.rb b/lib/backup/database_model.rb index 6129a3ce891..b2202ad7794 100644 --- a/lib/backup/database_model.rb +++ b/lib/backup/database_model.rb @@ -16,6 +16,8 @@ module Backup sslcompression: 'PGSSLCOMPRESSION' }.freeze + OVERRIDE_PREFIX = "GITLAB_BACKUP_" + attr_reader :config def initialize(name) @@ -35,7 +37,7 @@ module Backup original_config = source_model.connection_db_config.configuration_hash.dup - @config = config_for_backup(original_config) + @config = config_for_backup(name, original_config) @model.establish_connection( ActiveRecord::DatabaseConfigurations::HashConfig.new( @@ -56,7 +58,7 @@ module Backup self.class.const_set(klass_name, Class.new(ApplicationRecord)) end - def config_for_backup(config) + def config_for_backup(name, config) db_config = { activerecord: config, pg_env: {} @@ -65,8 +67,9 @@ module Backup # This enables the use of different PostgreSQL settings in # case PgBouncer is used. PgBouncer clears the search path, # which wreaks havoc on Rails if connections are reused. - override = "GITLAB_BACKUP_#{arg}" - val = ENV[override].presence || config[opt].to_s.presence + override_all = "#{OVERRIDE_PREFIX}#{arg}" + override_db = "#{OVERRIDE_PREFIX}#{name.upcase}_#{arg}" + val = ENV[override_db].presence || ENV[override_all].presence || config[opt].to_s.presence next unless val diff --git a/lib/backup/files.rb b/lib/backup/files.rb index 9b019f16ddd..b8ff7fff591 100644 --- a/lib/backup/files.rb +++ b/lib/backup/files.rb @@ -53,7 +53,7 @@ module Backup end override :restore - def restore(backup_tarball) + def restore(backup_tarball, backup_id) backup_existing_files_dir(backup_tarball) cmd_list = [%w[gzip -cd], %W[#{tar} --unlink-first --recursive-unlink -C #{app_files_realpath} -xf -]] diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 2cded4a55bb..1c53e675b2a 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -68,7 +68,7 @@ module Backup end puts_time "Dumping #{definition.human_name} ... ".color(:blue) - definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path), full_backup_id) + definition.task.dump(File.join(Gitlab.config.backup.path, definition.destination_path), backup_id) puts_time "Dumping #{definition.human_name} ... ".color(:blue) + "done".color(:green) rescue Backup::DatabaseBackupError, Backup::FileBackupError => e @@ -102,7 +102,7 @@ module Backup Gitlab::TaskHelpers.ask_to_continue end - definition.task.restore(File.join(Gitlab.config.backup.path, definition.destination_path)) + definition.task.restore(File.join(Gitlab.config.backup.path, definition.destination_path), backup_id) puts_time "Restoring #{definition.human_name} ... ".color(:blue) + "done".color(:green) diff --git a/lib/backup/repositories.rb b/lib/backup/repositories.rb index 3b1547148d8..46825dbd203 100644 --- a/lib/backup/repositories.rb +++ b/lib/backup/repositories.rb @@ -31,8 +31,8 @@ module Backup end override :restore - def restore(destination_path) - strategy.start(:restore, destination_path, remove_all_repositories: remove_all_repositories) + def restore(destination_path, backup_id) + strategy.start(:restore, destination_path, remove_all_repositories: remove_all_repositories, backup_id: backup_id) enqueue_consecutive ensure @@ -58,11 +58,8 @@ module Backup end def enqueue_consecutive_projects - cross_join_issue = "https://gitlab.com/gitlab-org/gitlab/-/issues/417467" - ::Gitlab::Database.allow_cross_joins_across_databases(url: cross_join_issue) do - project_relation.find_each(batch_size: 1000) do |project| - enqueue_project(project) - end + project_relation.find_each(batch_size: 1000) do |project| + enqueue_project(project) end end @@ -84,7 +81,7 @@ module Backup end def project_relation - scope = Project.includes(:route, :group, namespace: :owner) + scope = Project.includes(:route, :group, :namespace) scope = scope.id_in(ProjectRepository.for_repository_storage(storages).select(:project_id)) if storages.any? if paths.any? scope = scope.where_full_path_in(paths).or( diff --git a/lib/backup/task.rb b/lib/backup/task.rb index 776c19130a7..65059f3a3cb 100644 --- a/lib/backup/task.rb +++ b/lib/backup/task.rb @@ -15,7 +15,7 @@ module Backup end # restore task backup from `path` - def restore(path) + def restore(path, backup_id) raise NotImplementedError end diff --git a/lib/banzai/color_parser.rb b/lib/banzai/color_parser.rb index cce79e73d2d..6d01d51955c 100644 --- a/lib/banzai/color_parser.rb +++ b/lib/banzai/color_parser.rb @@ -2,13 +2,13 @@ module Banzai module ColorParser - ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/.freeze # 0.0..1.0 - PERCENTS = /(?:\d{1,2}|100)%/.freeze # 00%..100% - ALPHA_CHANNEL = /(?:,\s*(?:#{ALPHA}|#{PERCENTS}))?/.freeze - BITS = /\d{1,2}|1\d\d|2(?:[0-4]\d|5[0-5])/.freeze # 00..255 - DEGS = /-?\d+(?:deg)?/i.freeze # [-]digits[deg] - RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i.freeze # [-](digits[.digits] OR .digits)rad - HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/.freeze + ALPHA = /0(?:\.\d+)?|\.\d+|1(?:\.0+)?/ # 0.0..1.0 + PERCENTS = /(?:\d{1,2}|100)%/ # 00%..100% + ALPHA_CHANNEL = /(?:,\s*(?:#{ALPHA}|#{PERCENTS}))?/ + BITS = /\d{1,2}|1\d\d|2(?:[0-4]\d|5[0-5])/ # 00..255 + DEGS = /-?\d+(?:deg)?/i # [-]digits[deg] + RADS = /-?(?:\d+(?:\.\d+)?|\.\d+)rad/i # [-](digits[.digits] OR .digits)rad + HEX_FORMAT = /\#(?:\h{3}|\h{4}|\h{6}|\h{8})/ RGB_FORMAT = %r{ (?:rgba? \( @@ -20,7 +20,7 @@ module Banzai #{ALPHA_CHANNEL} \) ) - }xi.freeze + }xi HSL_FORMAT = %r{ (?:hsla? \( @@ -28,11 +28,11 @@ module Banzai #{ALPHA_CHANNEL} \) ) - }xi.freeze + }xi FORMATS = [HEX_FORMAT, RGB_FORMAT, HSL_FORMAT].freeze - COLOR_FORMAT = /\A(#{Regexp.union(FORMATS)})\z/ix.freeze + COLOR_FORMAT = /\A(#{Regexp.union(FORMATS)})\z/ix # Public: Analyzes whether the String is a color code. # diff --git a/lib/banzai/filter/ascii_doc_sanitization_filter.rb b/lib/banzai/filter/ascii_doc_sanitization_filter.rb index 4158aa8a5ec..3d425b9795f 100644 --- a/lib/banzai/filter/ascii_doc_sanitization_filter.rb +++ b/lib/banzai/filter/ascii_doc_sanitization_filter.rb @@ -7,7 +7,7 @@ module Banzai # Extends Banzai::Filter::BaseSanitizationFilter with specific rules. class AsciiDocSanitizationFilter < Banzai::Filter::BaseSanitizationFilter # Anchor link prefixed by "user-content-" pattern - PREFIXED_ID_PATTERN = /\A#{Gitlab::Asciidoc::DEFAULT_ADOC_ATTRS['idprefix']}(:?[[:alnum:]]|-|_)+\z/.freeze + PREFIXED_ID_PATTERN = /\A#{Gitlab::Asciidoc::DEFAULT_ADOC_ATTRS['idprefix']}(:?[[:alnum:]]|-|_)+\z/ SECTION_HEADINGS = %w[h2 h3 h4 h5 h6].freeze # Footnote link patterns diff --git a/lib/banzai/filter/attributes_filter.rb b/lib/banzai/filter/attributes_filter.rb index ab50b3d6858..98b0ed8cc22 100644 --- a/lib/banzai/filter/attributes_filter.rb +++ b/lib/banzai/filter/attributes_filter.rb @@ -16,9 +16,9 @@ module Banzai CSS = 'img' XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze - ATTRIBUTES_PATTERN = %r{\A(?<matched>\{(?<attributes>.{1,100})\})}.freeze - WIDTH_HEIGHT_REGEX = %r{\A(?<name>height|width)="?(?<size>[\w%]{1,10})"?\z}.freeze - VALID_SIZE_REGEX = %r{\A\d{1,4}(%|px)?\z}.freeze + ATTRIBUTES_PATTERN = %r{\A(?<matched>\{(?<attributes>.{1,100})\})} + WIDTH_HEIGHT_REGEX = %r{\A(?<name>height|width)="?(?<size>[\w%]{1,10})"?\z} + VALID_SIZE_REGEX = %r{\A\d{1,4}(%|px)?\z} def call doc.xpath(XPATH).each do |img| diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index e877e5f316c..87e2d94af94 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -36,7 +36,7 @@ module Banzai # Rubular: http://rubular.com/r/nrL3r9yUiq # Note that it's not possible to use Gitlab::UntrustedRegexp for LINK_PATTERN, # as `(?<!` is unsupported in `re2`, see https://github.com/google/re2/wiki/Syntax - LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!\?|!|\.|,|:)}.freeze + LINK_PATTERN = %r{([a-z][a-z0-9\+\.-]+://[^\s>]+)(?<!\?|!|\.|,|:)} ENTITY_UNTRUSTED = '((?:&[\w#]+;)+)\z' ENTITY_UNTRUSTED_REGEX = Gitlab::UntrustedRegexp.new(ENTITY_UNTRUSTED, multiline: false) diff --git a/lib/banzai/filter/blockquote_fence_filter.rb b/lib/banzai/filter/blockquote_fence_filter.rb index d4ff7d4c6b5..34b9fd63b1c 100644 --- a/lib/banzai/filter/blockquote_fence_filter.rb +++ b/lib/banzai/filter/blockquote_fence_filter.rb @@ -32,7 +32,7 @@ module Banzai ) \n\ *>>>\ *(?=\n$|\z) ) - }mx.freeze + }mx def initialize(text, context = nil, result = nil) super text, context, result diff --git a/lib/banzai/filter/footnote_filter.rb b/lib/banzai/filter/footnote_filter.rb index f10efdccdf1..ada74d613f9 100644 --- a/lib/banzai/filter/footnote_filter.rb +++ b/lib/banzai/filter/footnote_filter.rb @@ -19,8 +19,8 @@ module Banzai class FootnoteFilter < HTML::Pipeline::Filter FOOTNOTE_ID_PREFIX = 'fn-' FOOTNOTE_LINK_ID_PREFIX = 'fnref-' - FOOTNOTE_LI_REFERENCE_PATTERN = /\A#{FOOTNOTE_ID_PREFIX}.+\z/.freeze - FOOTNOTE_LINK_REFERENCE_PATTERN = /\A#{FOOTNOTE_LINK_ID_PREFIX}.+\z/.freeze + FOOTNOTE_LI_REFERENCE_PATTERN = /\A#{FOOTNOTE_ID_PREFIX}.+\z/ + FOOTNOTE_LINK_REFERENCE_PATTERN = /\A#{FOOTNOTE_LINK_ID_PREFIX}.+\z/ CSS_SECTION = "section[data-footnotes]" XPATH_SECTION = Gitlab::Utils::Nokogiri.css_to_xpath(CSS_SECTION).freeze diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index ade4f82e54b..9c3ad4c6a0c 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -51,10 +51,10 @@ module Banzai # See https://github.com/gollum/gollum/wiki # # Rubular: http://rubular.com/r/7dQnE5CUCH - TAGS_PATTERN = /\[\[(.+?)\]\]/.freeze + TAGS_PATTERN = /\[\[(.+?)\]\]/ # Pattern to match allowed image extensions - ALLOWED_IMAGE_EXTENSIONS = /.+(jpg|png|gif|svg|bmp)\z/i.freeze + ALLOWED_IMAGE_EXTENSIONS = /.+(jpg|png|gif|svg|bmp)\z/i # Do not perform linking inside these tags. IGNORED_ANCESTOR_TAGS = %w[pre code tt].to_set diff --git a/lib/banzai/filter/inline_observability_filter.rb b/lib/banzai/filter/inline_observability_filter.rb deleted file mode 100644 index 8e38f689959..00000000000 --- a/lib/banzai/filter/inline_observability_filter.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -module Banzai - module Filter - class InlineObservabilityFilter < HTML::Pipeline::Filter - include Gitlab::Utils::StrongMemoize - - def call - return doc unless Gitlab::Observability.enabled?(group) - - doc.xpath(xpath_search).each do |node| - next unless element = element_to_embed(node) - - # We want this to follow any surrounding content. For example, - # if a link is inline in a paragraph. - node.parent.children.last.add_next_sibling(element) - end - - doc - end - - # Placeholder element for the frontend to use as an - # injection point for observability. - def create_element(url) - doc.document.create_element( - 'div', - class: 'js-render-observability', - 'data-frame-url': url - ) - end - - # Search params for selecting observability links. - def xpath_search - "descendant-or-self::a[starts-with(@href, '#{gitlab_domain}/groups/') and contains(@href,'/-/observability/')]" - end - - # Creates a new element based on the parameters - # obtained from the target link - def element_to_embed(node) - url = node['href'] - - embeddable_url = extract_embeddable_url(url) - create_element(embeddable_url) if embeddable_url - end - - private - - def extract_embeddable_url(url) - strong_memoize_with(:embeddable_url, url) do - Gitlab::Observability.embeddable_url(url) - end - end - - def group - context[:group] || context[:project]&.group - end - - def gitlab_domain - ::Gitlab.config.gitlab.url - end - end - end -end diff --git a/lib/banzai/filter/markdown_post_escape_filter.rb b/lib/banzai/filter/markdown_post_escape_filter.rb index 4d37fba33aa..90b9555df1d 100644 --- a/lib/banzai/filter/markdown_post_escape_filter.rb +++ b/lib/banzai/filter/markdown_post_escape_filter.rb @@ -5,9 +5,9 @@ module Banzai # See comments in MarkdownPreEscapeFilter for details on strategy class MarkdownPostEscapeFilter < HTML::Pipeline::Filter LITERAL_KEYWORD = MarkdownPreEscapeFilter::LITERAL_KEYWORD - LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-(.*?)-#{LITERAL_KEYWORD}}.freeze - NOT_LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-((%5C|\\).+?)-#{LITERAL_KEYWORD}}.freeze - SPAN_REGEX = %r{<span data-escaped-char>(.*?)</span>}.freeze + LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-(.*?)-#{LITERAL_KEYWORD}} + NOT_LITERAL_REGEX = %r{#{LITERAL_KEYWORD}-((%5C|\\).+?)-#{LITERAL_KEYWORD}} + SPAN_REGEX = %r{<span data-escaped-char>(.*?)</span>} XPATH_A = Gitlab::Utils::Nokogiri.css_to_xpath('a').freeze XPATH_LANG_TAG = Gitlab::Utils::Nokogiri.css_to_xpath('pre').freeze diff --git a/lib/banzai/filter/markdown_pre_escape_filter.rb b/lib/banzai/filter/markdown_pre_escape_filter.rb index 8cc7b0defd6..b6f063ece57 100644 --- a/lib/banzai/filter/markdown_pre_escape_filter.rb +++ b/lib/banzai/filter/markdown_pre_escape_filter.rb @@ -57,7 +57,7 @@ module Banzai ].freeze TARGET_CHARS = ESCAPABLE_CHARS.pluck(:char).join.freeze - ASCII_PUNCTUATION = %r{(\\[#{TARGET_CHARS}])}.freeze + ASCII_PUNCTUATION = %r{(\\[#{TARGET_CHARS}])} LITERAL_KEYWORD = 'cmliteral' def call diff --git a/lib/banzai/filter/math_filter.rb b/lib/banzai/filter/math_filter.rb index e568f51652f..3161e030194 100644 --- a/lib/banzai/filter/math_filter.rb +++ b/lib/banzai/filter/math_filter.rb @@ -47,7 +47,7 @@ module Banzai # Add necessary classes to any existing math blocks def process_existing doc.xpath(XPATH_INLINE_CODE).each do |code| - break if @nodes_count >= RENDER_NODES_LIMIT + break if render_nodes_limit_reached?(@nodes_count) code[:class] = MATH_CLASSES @@ -58,7 +58,7 @@ module Banzai # Corresponds to the "$`...`$" syntax def process_dollar_backtick_inline doc.xpath(XPATH_CODE).each do |code| - break if @nodes_count >= RENDER_NODES_LIMIT + break if render_nodes_limit_reached?(@nodes_count) closing = code.next opening = code.previous @@ -87,6 +87,16 @@ module Banzai pre_node[:class] = TAG_CLASS end end + + def settings + Gitlab::CurrentSettings.current_application_settings + end + + def render_nodes_limit_reached?(count) + return false unless settings.math_rendering_limits_enabled? + + count >= RENDER_NODES_LIMIT + end end end end diff --git a/lib/banzai/filter/references/abstract_reference_filter.rb b/lib/banzai/filter/references/abstract_reference_filter.rb index 3e48fe33b03..c3c5103106b 100644 --- a/lib/banzai/filter/references/abstract_reference_filter.rb +++ b/lib/banzai/filter/references/abstract_reference_filter.rb @@ -20,7 +20,7 @@ module Banzai # transitory value (it never gets saved) we can initialize once, and it # doesn't matter if it changes on a restart. REFERENCE_PLACEHOLDER = "_reference_#{SecureRandom.hex(16)}_" - REFERENCE_PLACEHOLDER_PATTERN = %r{#{REFERENCE_PLACEHOLDER}(\d+)}.freeze + REFERENCE_PLACEHOLDER_PATTERN = %r{#{REFERENCE_PLACEHOLDER}(\d+)} # Public: Find references in text (like `!123` for merge requests) # diff --git a/lib/banzai/filter/references/reference_filter.rb b/lib/banzai/filter/references/reference_filter.rb index caec808ef04..e7fa287ae06 100644 --- a/lib/banzai/filter/references/reference_filter.rb +++ b/lib/banzai/filter/references/reference_filter.rb @@ -153,6 +153,7 @@ module Banzai @ignore_ancestor_query ||= begin parents = %w[pre code a style] parents << 'blockquote' if context[:ignore_blockquotes] + parents << 'span[contains(concat(" ", @class, " "), " idiff ")]' parents.map { |n| "ancestor::#{n}" }.join(' or ') end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index 15013c8595e..f33bc9cd621 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -7,7 +7,7 @@ module Banzai # Extends Banzai::Filter::BaseSanitizationFilter with specific rules. class SanitizationFilter < Banzai::Filter::BaseSanitizationFilter # Styles used by Markdown for table alignment - TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/.freeze + TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/ def customize_allowlist(allowlist) allowlist[:allow_comments] = context[:allow_comments] diff --git a/lib/banzai/filter/task_list_filter.rb b/lib/banzai/filter/task_list_filter.rb index e8a7677b102..4f39a25ff68 100644 --- a/lib/banzai/filter/task_list_filter.rb +++ b/lib/banzai/filter/task_list_filter.rb @@ -32,7 +32,7 @@ module Banzai XPATH = 'descendant-or-self::li[input[@data-inapplicable]] | descendant-or-self::li[p[input[@data-inapplicable]]]' INAPPLICABLE = '[~]' - INAPPLICABLEPATTERN = /\[~\]/.freeze + INAPPLICABLEPATTERN = /\[~\]/ # Pattern used to identify all task list items. # Useful when you need iterate over all items. @@ -46,7 +46,7 @@ module Banzai #{INAPPLICABLEPATTERN} ) (?=\s) # followed by whitespace - /x.freeze + /x # Force the gem's constant to use our new one superclass.send(:remove_const, :ItemPattern) # rubocop: disable GitlabSecurity/PublicSend diff --git a/lib/banzai/pipeline/gfm_pipeline.rb b/lib/banzai/pipeline/gfm_pipeline.rb index 1d6269c704d..1b3905f0dde 100644 --- a/lib/banzai/pipeline/gfm_pipeline.rb +++ b/lib/banzai/pipeline/gfm_pipeline.rb @@ -32,12 +32,11 @@ module Banzai Filter::ExternalLinkFilter, Filter::SuggestionFilter, Filter::FootnoteFilter, + Filter::InlineDiffFilter, *reference_filters, Filter::EmojiFilter, Filter::CustomEmojiFilter, Filter::TaskListFilter, - Filter::InlineDiffFilter, - Filter::InlineObservabilityFilter, Filter::SetDirectionFilter, Filter::SyntaxHighlightFilter # this filter should remain at the end ] diff --git a/lib/bitbucket/representation/issue.rb b/lib/bitbucket/representation/issue.rb index 7a50bdf58d6..e9a2fc7510f 100644 --- a/lib/bitbucket/representation/issue.rb +++ b/lib/bitbucket/representation/issue.rb @@ -45,6 +45,19 @@ module Bitbucket iid end + def to_hash + { + iid: iid, + title: title, + description: description, + state: state, + author: author, + milestone: milestone, + created_at: created_at, + updated_at: updated_at + } + end + private def closed? diff --git a/lib/bitbucket/representation/pull_request.rb b/lib/bitbucket/representation/pull_request.rb index 6451b8f5d1f..ab8f5ba17fe 100644 --- a/lib/bitbucket/representation/pull_request.rb +++ b/lib/bitbucket/representation/pull_request.rb @@ -58,6 +58,10 @@ module Bitbucket raw['reviewers']&.pluck('username') end + def merge_commit_sha + raw['merge_commit']&.dig('hash') + end + def to_hash { iid: iid, @@ -76,10 +80,6 @@ module Bitbucket } end - def merge_commit_sha - raw['merge_commit']&.dig('hash') - end - private def source_branch diff --git a/lib/bitbucket_server/representation/pull_request.rb b/lib/bitbucket_server/representation/pull_request.rb index 66dba5fefc7..996a10318f5 100644 --- a/lib/bitbucket_server/representation/pull_request.rb +++ b/lib/bitbucket_server/representation/pull_request.rb @@ -44,6 +44,10 @@ module BitbucketServer state == 'merged' end + def closed? + state == 'closed' + end + def created_at self.class.convert_timestamp(created_date) end diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index c9ed75e663e..6c2aa41c346 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -138,7 +138,6 @@ module BulkImports def default_options { - headers: { 'Content-Type' => 'application/json' }, query: request_query, follow_redirects: true, resend_on_redirect: false, diff --git a/lib/bulk_imports/common/pipelines/badges_pipeline.rb b/lib/bulk_imports/common/pipelines/badges_pipeline.rb index 33a24e61a3f..8259a90292f 100644 --- a/lib/bulk_imports/common/pipelines/badges_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/badges_pipeline.rb @@ -5,6 +5,7 @@ module BulkImports module Pipelines class BadgesPipeline include Pipeline + include HexdigestCacheStrategy extractor BulkImports::Common::Extractors::RestExtractor, query: BulkImports::Common::Rest::GetBadgesQuery diff --git a/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb b/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb index bd09b6add00..ab12c590e54 100644 --- a/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/lfs_objects_pipeline.rb @@ -5,6 +5,7 @@ module BulkImports module Pipelines class LfsObjectsPipeline include Pipeline + include IndexCacheStrategy file_extraction_pipeline! diff --git a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb index ea17af36c9a..bc42ddc59ca 100644 --- a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb @@ -5,8 +5,9 @@ module BulkImports module Pipelines class UploadsPipeline include Pipeline + include IndexCacheStrategy - AVATAR_PATTERN = %r{.*\/#{BulkImports::UploadsExportService::AVATAR_PATH}\/(?<identifier>.*)}.freeze + AVATAR_PATTERN = %r{.*\/#{BulkImports::UploadsExportService::AVATAR_PATH}\/(?<identifier>.*)} AvatarLoadingError = Class.new(StandardError) diff --git a/lib/bulk_imports/common/transformers/user_reference_transformer.rb b/lib/bulk_imports/common/transformers/user_reference_transformer.rb deleted file mode 100644 index c330ea59113..00000000000 --- a/lib/bulk_imports/common/transformers/user_reference_transformer.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -# UserReferenceTransformer replaces specified user -# reference key with a user id being either: -# - A user id found by `public_email` in the group -# - Current user id -# under a new key `"#{@reference}_id"`. -module BulkImports - module Common - module Transformers - class UserReferenceTransformer - DEFAULT_REFERENCE = 'user' - - def initialize(options = {}) - @reference = options[:reference].to_s.presence || DEFAULT_REFERENCE - @suffixed_reference = "#{@reference}_id" - end - - def transform(context, data) - return unless data - - user = find_user(context, data&.dig(@reference, 'public_email')) || context.current_user - - data - .except(@reference) - .merge(@suffixed_reference => user.id) - end - - private - - def find_user(context, email) - return if email.blank? - - context.group.users.find_by_any_email(email, confirmed: true) # rubocop: disable CodeReuse/ActiveRecord - end - end - end - end -end diff --git a/lib/bulk_imports/file_downloads/filename_fetch.rb b/lib/bulk_imports/file_downloads/filename_fetch.rb index b6bb0fd8c81..ac58e0f8fd6 100644 --- a/lib/bulk_imports/file_downloads/filename_fetch.rb +++ b/lib/bulk_imports/file_downloads/filename_fetch.rb @@ -3,7 +3,7 @@ module BulkImports module FileDownloads module FilenameFetch - REMOTE_FILENAME_PATTERN = %r{filename="(?<filename>[^"]+)"}.freeze + REMOTE_FILENAME_PATTERN = %r{filename="(?<filename>[^"]+)"} FILENAME_SIZE_LIMIT = 255 # chars before the extension def raise_error(message) diff --git a/lib/bulk_imports/groups/pipelines/group_pipeline.rb b/lib/bulk_imports/groups/pipelines/group_pipeline.rb index 8c6f089e8a4..a98d382189f 100644 --- a/lib/bulk_imports/groups/pipelines/group_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/group_pipeline.rb @@ -5,6 +5,7 @@ module BulkImports module Pipelines class GroupPipeline include Pipeline + include HexdigestCacheStrategy abort_on_failure! diff --git a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb index 7d9d8120e6c..6f8e2e2d8d9 100644 --- a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb @@ -5,6 +5,7 @@ module BulkImports module Pipelines class ProjectEntitiesPipeline include Pipeline + include HexdigestCacheStrategy extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetProjectsQuery transformer Common::Transformers::ProhibitedAttributesTransformer diff --git a/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb index c47a8bd1daa..3b7374bb90e 100644 --- a/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb @@ -5,6 +5,7 @@ module BulkImports module Pipelines class SubgroupEntitiesPipeline include Pipeline + include HexdigestCacheStrategy extractor BulkImports::Groups::Extractors::SubgroupsExtractor transformer Common::Transformers::ProhibitedAttributesTransformer diff --git a/lib/bulk_imports/ndjson_pipeline.rb b/lib/bulk_imports/ndjson_pipeline.rb index 3c392910c1f..89ae66938af 100644 --- a/lib/bulk_imports/ndjson_pipeline.rb +++ b/lib/bulk_imports/ndjson_pipeline.rb @@ -5,6 +5,7 @@ module BulkImports extend ActiveSupport::Concern include Pipeline + include Pipeline::IndexCacheStrategy included do file_extraction_pipeline! diff --git a/lib/bulk_imports/pipeline/extracted_data.rb b/lib/bulk_imports/pipeline/extracted_data.rb index 0b36c068298..e4640db0873 100644 --- a/lib/bulk_imports/pipeline/extracted_data.rb +++ b/lib/bulk_imports/pipeline/extracted_data.rb @@ -5,6 +5,8 @@ module BulkImports class ExtractedData attr_reader :data + delegate :each, :each_with_index, to: :data + def initialize(data: nil, page_info: {}) @data = data.is_a?(Enumerator) ? data : Array.wrap(data) @page_info = page_info @@ -20,10 +22,6 @@ module BulkImports def next_page @page_info&.dig('next_page') end - - def each(&block) - data.each(&block) - end end end end diff --git a/lib/bulk_imports/pipeline/hexdigest_cache_strategy.rb b/lib/bulk_imports/pipeline/hexdigest_cache_strategy.rb new file mode 100644 index 00000000000..51d7374f6c6 --- /dev/null +++ b/lib/bulk_imports/pipeline/hexdigest_cache_strategy.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module BulkImports + module Pipeline + module HexdigestCacheStrategy + def already_processed?(data, _) + values = Gitlab::Cache::Import::Caching.values_from_set(cache_key) + values.include?(OpenSSL::Digest::SHA256.hexdigest(data.to_s)) + end + + def save_processed_entry(data, _) + Gitlab::Cache::Import::Caching.set_add(cache_key, OpenSSL::Digest::SHA256.hexdigest(data.to_s)) + end + end + end +end diff --git a/lib/bulk_imports/pipeline/index_cache_strategy.rb b/lib/bulk_imports/pipeline/index_cache_strategy.rb new file mode 100644 index 00000000000..7d5ab1148e8 --- /dev/null +++ b/lib/bulk_imports/pipeline/index_cache_strategy.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module BulkImports + module Pipeline + module IndexCacheStrategy + def already_processed?(_, index) + last_index = Gitlab::Cache::Import::Caching.read(cache_key) + last_index && last_index.to_i >= index + end + + def save_processed_entry(_, index) + Gitlab::Cache::Import::Caching.write(cache_key, index) + end + end + end +end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index 1e2d9152047..666916f8758 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -15,7 +15,10 @@ module BulkImports extracted_data = extracted_data_from if extracted_data - extracted_data.each do |entry| + extracted_data.each_with_index do |entry, index| + raw_entry = entry.dup + next if Feature.enabled?(:bulk_import_idempotent_workers) && already_processed?(raw_entry, index) + transformers.each do |transformer| entry = run_pipeline_step(:transformer, transformer.class.name) do transformer.transform(context, entry) @@ -25,6 +28,8 @@ module BulkImports run_pipeline_step(:loader, loader.class.name) do loader.load(context, entry) end + + save_processed_entry(raw_entry, index) if Feature.enabled?(:bulk_import_idempotent_workers) end tracker.update!( @@ -73,6 +78,19 @@ module BulkImports end end + def cache_key + batch_number = context.extra[:batch_number] || 0 + + "#{self.class.name.underscore}/#{tracker.bulk_import_entity_id}/#{batch_number}" + end + + # Overridden by child pipelines with different caching strategies + def already_processed?(*) + false + end + + def save_processed_entry(*); end + def after_run(extracted_data) run if extracted_data.has_next_page? end @@ -80,9 +98,9 @@ module BulkImports def log_and_fail(exception, step) log_import_failure(exception, step) - tracker.fail_op! - if abort_on_failure? + tracker.fail_op! + warn(message: 'Aborting entity migration due to pipeline failure') context.entity.fail_op! end @@ -112,7 +130,7 @@ module BulkImports { bulk_import_id: context.bulk_import_id, pipeline_step: step, - message: 'Pipeline failed' + message: 'An object of a pipeline failed to import' } ) ) diff --git a/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb b/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb index 235d2629b9e..af41eea3135 100644 --- a/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb @@ -5,6 +5,7 @@ module BulkImports module Pipelines class DesignBundlePipeline include Pipeline + include IndexCacheStrategy file_extraction_pipeline! relation_name BulkImports::FileTransfer::ProjectConfig::DESIGN_BUNDLE_RELATION diff --git a/lib/bulk_imports/projects/pipelines/project_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_pipeline.rb index c9da33fe8e3..22384a96aa5 100644 --- a/lib/bulk_imports/projects/pipelines/project_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/project_pipeline.rb @@ -5,6 +5,7 @@ module BulkImports module Pipelines class ProjectPipeline include Pipeline + include HexdigestCacheStrategy abort_on_failure! diff --git a/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb b/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb index 4307cb2bafd..eb6aa0c0858 100644 --- a/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb @@ -5,6 +5,7 @@ module BulkImports module Pipelines class RepositoryBundlePipeline include Pipeline + include IndexCacheStrategy abort_on_failure! file_extraction_pipeline! diff --git a/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb b/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb index e29601927be..39c9c121797 100644 --- a/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb @@ -5,6 +5,7 @@ module BulkImports module Pipelines class SnippetsRepositoryPipeline include Pipeline + include HexdigestCacheStrategy extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetSnippetRepositoryQuery diff --git a/lib/container_registry/base_client.rb b/lib/container_registry/base_client.rb index 0b24b31c4ae..ac7b8d3b455 100644 --- a/lib/container_registry/base_client.rb +++ b/lib/container_registry/base_client.rb @@ -47,11 +47,15 @@ module ContainerRegistry def token_from(config) case config[:type] when :full_access_token - Auth::ContainerRegistryAuthenticationService.access_token([], []) + Auth::ContainerRegistryAuthenticationService.access_token({}) when :nested_repositories_token return unless config[:path] Auth::ContainerRegistryAuthenticationService.pull_nested_repositories_access_token(config[:path]) + when :push_pull_nested_repositories_token + return unless config[:path] + + Auth::ContainerRegistryAuthenticationService.push_pull_nested_repositories_access_token(config[:path]) end end end diff --git a/lib/container_registry/client.rb b/lib/container_registry/client.rb index b83d67c359d..e2a1b8296f6 100644 --- a/lib/container_registry/client.rb +++ b/lib/container_registry/client.rb @@ -10,6 +10,7 @@ module ContainerRegistry REGISTRY_VERSION_HEADER = 'gitlab-container-registry-version' REGISTRY_FEATURES_HEADER = 'gitlab-container-registry-features' REGISTRY_TAG_DELETE_FEATURE = 'tag_delete' + REGISTRY_DB_ENABLED_HEADER = 'gitlab-container-registry-database-enabled' DEFAULT_TAGS_PAGE_SIZE = 10000 @@ -47,11 +48,13 @@ module ContainerRegistry version = response.headers[REGISTRY_VERSION_HEADER] features = response.headers.fetch(REGISTRY_FEATURES_HEADER, '') + db_enabled = response.headers.fetch(REGISTRY_DB_ENABLED_HEADER, '') { version: version, features: features.split(',').map(&:strip), - vendor: version ? 'gitlab' : 'other' + vendor: version ? 'gitlab' : 'other', + db_enabled: ::Gitlab::Utils.to_boolean(db_enabled, default: false) } end diff --git a/lib/container_registry/gitlab_api_client.rb b/lib/container_registry/gitlab_api_client.rb index ab7566119c4..bd833ec00af 100644 --- a/lib/container_registry/gitlab_api_client.rb +++ b/lib/container_registry/gitlab_api_client.rb @@ -6,6 +6,7 @@ module ContainerRegistry JSON_TYPE = 'application/json' CANCEL_RESPONSE_STATUS_HEADER = 'status' + GITLAB_REPOSITORIES_PATH = '/gitlab/v1/repositories' IMPORT_RESPONSES = { 200 => :already_imported, @@ -19,6 +20,16 @@ module ContainerRegistry 429 => :too_many_imports }.freeze + RENAME_RESPONSES = { + 202 => :accepted, + 204 => :ok, + 400 => :bad_request, + 401 => :unauthorized, + 404 => :not_found, + 409 => :name_taken, + 422 => :too_many_subrepositories + }.freeze + REGISTRY_GITLAB_V1_API_FEATURE = 'gitlab_v1_api' MAX_TAGS_PAGE_SIZE = 1000 @@ -34,14 +45,16 @@ module ContainerRegistry end def self.deduplicated_size(path) - with_dummy_client(token_config: { type: :nested_repositories_token, path: path&.downcase }) do |client| - client.repository_details(path&.downcase, sizing: :self_with_descendants)['size_bytes'] + downcased_path = path&.downcase + with_dummy_client(token_config: { type: :nested_repositories_token, path: downcased_path }) do |client| + client.repository_details(downcased_path, sizing: :self_with_descendants)['size_bytes'] end end def self.one_project_with_container_registry_tag(path) - with_dummy_client(token_config: { type: :nested_repositories_token, path: path&.downcase }) do |client| - page = client.sub_repositories_with_tag(path&.downcase, page_size: PAGE_SIZE) + downcased_path = path&.downcase + with_dummy_client(token_config: { type: :nested_repositories_token, path: downcased_path }) do |client| + page = client.sub_repositories_with_tag(downcased_path, page_size: PAGE_SIZE) details = page[:response_body]&.first break unless details @@ -54,17 +67,26 @@ module ContainerRegistry end end + def self.rename_base_repository_path(path, name:, dry_run: false) + downcased_path = path&.downcase + + with_dummy_client(token_config: { type: :push_pull_nested_repositories_token, path: downcased_path }) do |client| + client.rename_base_repository_path(downcased_path, name: name&.downcase, dry_run: dry_run) + end + end + def self.each_sub_repositories_with_tag_page(path:, page_size: 100, &block) raise ArgumentError, 'block not given' unless block # dummy uri to initialize the loop next_page_uri = URI('') page_count = 0 + downcased_path = path&.downcase - with_dummy_client(token_config: { type: :nested_repositories_token, path: path&.downcase }) do |client| + with_dummy_client(token_config: { type: :nested_repositories_token, path: downcased_path }) do |client| while next_page_uri last = Rack::Utils.parse_nested_query(next_page_uri.query)['last'] - current_page = client.sub_repositories_with_tag(path&.downcase, page_size: page_size, last: last) + current_page = client.sub_repositories_with_tag(downcased_path, page_size: page_size, last: last) if current_page&.key?(:response_body) yield (current_page[:response_body] || []) @@ -137,7 +159,7 @@ module ContainerRegistry # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#get-repository-details def repository_details(path, sizing: nil) with_token_faraday do |faraday_client| - req = faraday_client.get("/gitlab/v1/repositories/#{path}/") do |req| + req = faraday_client.get("#{GITLAB_REPOSITORIES_PATH}/#{path}/") do |req| req.params['size'] = sizing if sizing end @@ -151,7 +173,7 @@ module ContainerRegistry def tags(path, page_size: 100, last: nil, before: nil, name: nil, sort: nil) limited_page_size = [page_size, MAX_TAGS_PAGE_SIZE].min with_token_faraday do |faraday_client| - url = "/gitlab/v1/repositories/#{path}/tags/list/" + url = "#{GITLAB_REPOSITORIES_PATH}/#{path}/tags/list/" response = faraday_client.get(url) do |req| req.params['n'] = limited_page_size req.params['last'] = last if last @@ -211,6 +233,30 @@ module ContainerRegistry end end + # Given a path 'group/subgroup/project' and name 'newname', + # with a successful rename, it will be 'group/subgroup/newname' + # https://gitlab.com/gitlab-org/container-registry/-/blob/master/docs-gitlab/api.md#rename-base-repository + def rename_base_repository_path(path, name:, dry_run: false) + with_token_faraday do |faraday_client| + url = "#{GITLAB_REPOSITORIES_PATH}/#{path}/" + response = faraday_client.patch(url) do |req| + req.params['dry_run'] = dry_run + req.body = { name: name } + end + + unless response.success? + Gitlab::ErrorTracking.log_exception( + UnsuccessfulResponseError.new, + class: self.class.name, + url: url, + status_code: response.status + ) + end + + RENAME_RESPONSES.fetch(response.status, :error) + end + end + private def start_import_for(path, pre:) diff --git a/lib/error_tracking/sentry_client/pagination_parser.rb b/lib/error_tracking/sentry_client/pagination_parser.rb index c6a42a6def2..090707c21ab 100644 --- a/lib/error_tracking/sentry_client/pagination_parser.rb +++ b/lib/error_tracking/sentry_client/pagination_parser.rb @@ -3,7 +3,7 @@ module ErrorTracking class SentryClient module PaginationParser - PATTERN = /rel="(?<direction>\w+)";\sresults="(?<results>\w+)";\scursor="(?<cursor>.+)"/.freeze + PATTERN = /rel="(?<direction>\w+)";\sresults="(?<results>\w+)";\scursor="(?<cursor>.+)"/ def self.parse(headers) links = headers['link'].to_s.split(',') diff --git a/lib/expand_variables.rb b/lib/expand_variables.rb index 51a66958ba0..ad5aabfa1f3 100644 --- a/lib/expand_variables.rb +++ b/lib/expand_variables.rb @@ -1,18 +1,24 @@ # frozen_string_literal: true module ExpandVariables - VARIABLES_REGEXP = /\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/.freeze + VariableExpansionError = Class.new(StandardError) + + VARIABLES_REGEXP = /\$([a-zA-Z_][a-zA-Z0-9_]*)|\${\g<1>}|%\g<1>%/ class << self - def expand(value, variables, expand_file_refs: true) + def expand(value, variables, expand_file_refs: true, fail_on_masked: false) replace_with(value, variables) do |collection, last_match| - match_or_blank_value(collection, last_match, expand_file_refs: expand_file_refs) + match_or_blank_value( + collection, last_match, expand_file_refs: expand_file_refs, fail_on_masked: fail_on_masked + ) end end - def expand_existing(value, variables, expand_file_refs: true) + def expand_existing(value, variables, expand_file_refs: true, fail_on_masked: false) replace_with(value, variables) do |collection, last_match| - match_or_original_value(collection, last_match, expand_file_refs: expand_file_refs) + match_or_original_value( + collection, last_match, expand_file_refs: expand_file_refs, fail_on_masked: fail_on_masked + ) end end @@ -36,12 +42,14 @@ module ExpandVariables end end - def match_or_blank_value(collection, last_match, expand_file_refs:) + def match_or_blank_value(collection, last_match, expand_file_refs:, fail_on_masked:) match = last_match[1] || last_match[2] replacement = collection[match] if replacement.nil? nil + elsif fail_on_masked && replacement.masked? + raise VariableExpansionError, 'masked variables cannot be expanded' elsif replacement.file? expand_file_refs ? replacement.value : last_match else @@ -49,8 +57,10 @@ module ExpandVariables end end - def match_or_original_value(collection, last_match, expand_file_refs:) - match_or_blank_value(collection, last_match, expand_file_refs: expand_file_refs) || last_match[0] + def match_or_original_value(collection, last_match, expand_file_refs:, fail_on_masked:) + match_or_blank_value( + collection, last_match, expand_file_refs: expand_file_refs, fail_on_masked: fail_on_masked + ) || last_match[0] end end end diff --git a/lib/extracts_ref.rb b/lib/extracts_ref.rb index 49ec564eb8d..af3f841ea6e 100644 --- a/lib/extracts_ref.rb +++ b/lib/extracts_ref.rb @@ -1,32 +1,22 @@ # frozen_string_literal: true +# TOOD: https://gitlab.com/gitlab-org/gitlab/-/issues/425379 +# WARNING: This module has been deprecated. +# The module solely exists because ExtractsPath depends on this module (ExtractsPath is the only user.) +# ExtractsRef::RefExtractor class is a refactored version of this module and provides +# the same functionalities. You should use the class instead. +# # Module providing methods for dealing with separating a tree-ish string and a # file path string when combined in a request parameter # Can be extended for different types of repository object, e.g. Project or Snippet module ExtractsRef - InvalidPathError = Class.new(StandardError) - BRANCH_REF_TYPE = 'heads' - TAG_REF_TYPE = 'tags' - REF_TYPES = [BRANCH_REF_TYPE, TAG_REF_TYPE].freeze + InvalidPathError = ExtractsRef::RefExtractor::InvalidPathError + BRANCH_REF_TYPE = ExtractsRef::RefExtractor::BRANCH_REF_TYPE + TAG_REF_TYPE = ExtractsRef::RefExtractor::TAG_REF_TYPE + REF_TYPES = ExtractsRef::RefExtractor::REF_TYPES def self.ref_type(type) - return unless REF_TYPES.include?(type) - - type - end - - def self.qualify_ref(ref, type) - validated_type = ref_type(type) - return ref unless validated_type - - %(refs/#{validated_type}/#{ref}) - end - - def self.unqualify_ref(ref, type) - validated_type = ref_type(type) - return ref unless validated_type - - ref.sub(%r{^refs/#{validated_type}/}, '') + ExtractsRef::RefExtractor.ref_type(type) end # Given a string containing both a Git tree-ish, such as a branch or tag, and @@ -91,7 +81,7 @@ module ExtractsRef return unless @ref.present? @commit = if ref_type - @fully_qualified_ref = ExtractsRef.qualify_ref(@ref, ref_type) + @fully_qualified_ref = ExtractsRef::RefExtractor.qualify_ref(@ref, ref_type) @repo.commit(@fully_qualified_ref) else @repo.commit(@ref) diff --git a/lib/extracts_ref/ref_extractor.rb b/lib/extracts_ref/ref_extractor.rb new file mode 100644 index 00000000000..ac9b0ebb7af --- /dev/null +++ b/lib/extracts_ref/ref_extractor.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +# Module providing methods for dealing with separating a tree-ish string and a +# file path string when combined in a request parameter +# Can be extended for different types of repository object, e.g. Project or Snippet +module ExtractsRef + class RefExtractor + InvalidPathError = Class.new(StandardError) + BRANCH_REF_TYPE = 'heads' + TAG_REF_TYPE = 'tags' + REF_TYPES = [BRANCH_REF_TYPE, TAG_REF_TYPE].freeze + + attr_reader :repository_container, :params + attr_accessor :id, :ref, :commit, :path, :fully_qualified_ref + + class << self + def ref_type(type) + return unless REF_TYPES.include?(type&.downcase) + + type.downcase + end + + def qualify_ref(ref, type) + validated_type = ref_type(type) + return ref unless validated_type + + %(refs/#{validated_type}/#{ref}) + end + + def unqualify_ref(ref, type) + validated_type = ref_type(type) + return ref unless validated_type + + ref.sub(%r{^refs/#{validated_type}/}, '') + end + end + + def initialize(repository_container, params, override_id: nil) + @repository_container = repository_container + @params = params.extract!(:id, :ref, :path, :ref_type) + @override_id = override_id + end + + # Extracts common variables for views working with Git tree-ish objects + # + # Assignments are: + # + # - @id - A string representing the joined ref and path + # Assigns @override_id if it is present. + # - @ref - A string representing the ref (e.g., the branch, tag, or commit SHA) + # - @path - A string representing the filesystem path + # - @commit - A Commit representing the commit from the given ref + # - @fully_qualified_ref - A string representing the fully qualifed ref (e.g., refs/tags/v1.1) + # + # If the :id parameter appears to be requesting a specific response format, + # that will be handled as well. + def extract! + qualified_id, @ref, @path = extract_ref_path + @id = @override_id || qualified_id + @repo = repository_container.repository + raise InvalidPathError if @ref.match?(/\s/) + + return unless @ref.present? + + @commit = if ref_type + @fully_qualified_ref = self.class.qualify_ref(@ref, ref_type) + @repo.commit(@fully_qualified_ref) + else + @repo.commit(@ref) + end + end + + # Given a string containing both a Git tree-ish, such as a branch or tag, and + # a filesystem path joined by forward slashes, attempts to separate the two. + # + # Expects a repository_container method that returns the active repository object. This is + # used to check the input against a list of valid repository refs. + # + # Examples + # + # # No repository_container available + # extract_ref('master') + # # => ['', ''] + # + # extract_ref('master') + # # => ['master', ''] + # + # extract_ref("f4b14494ef6abf3d144c28e4af0c20143383e062/CHANGELOG") + # # => ['f4b14494ef6abf3d144c28e4af0c20143383e062', 'CHANGELOG'] + # + # extract_ref("v2.0.0/README.md") + # # => ['v2.0.0', 'README.md'] + # + # extract_ref('master/app/models/project.rb') + # # => ['master', 'app/models/project.rb'] + # + # extract_ref('issues/1234/app/models/project.rb') + # # => ['issues/1234', 'app/models/project.rb'] + # + # # Given an invalid branch, we fall back to just splitting on the first slash + # extract_ref('non/existent/branch/README.md') + # # => ['non', 'existent/branch/README.md'] + # + # Returns an Array where the first value is the tree-ish and the second is the + # path + def extract_ref(id) + pair = extract_raw_ref(id) + + [ + pair[0].strip, + pair[1].delete_prefix('/').delete_suffix('/') + ] + end + + def extract_ref_path + id = extract_id_from_params + ref, path = extract_ref(id) + + [id, ref, path] + end + + def ref_type + self.class.ref_type(params[:ref_type]) + end + + private + + def extract_raw_ref(id) + return ['', ''] unless repository_container + + # If the ref appears to be a SHA, we're done, just split the string + return $~.captures if id =~ /^(\h{40})(.+)/ + + # No slash means we must have a ref and no path + return [id, ''] unless id.include?('/') + + # Otherwise, attempt to detect the ref using a list of the + # repository_container's branches and tags + + # Append a trailing slash if we only get a ref and no file path + id = [id, '/'].join unless id.ends_with?('/') + first_path_segment, rest = id.split('/', 2) + + return [first_path_segment, rest] if use_first_path_segment?(first_path_segment) + + valid_refs = ref_names.select { |v| id.start_with?("#{v}/") } + + # No exact ref match, so just try our best + return id.match(%r{([^/]+)(.*)}).captures if valid_refs.empty? + + # There is a distinct possibility that multiple refs prefix the ID. + # Use the longest match to maximize the chance that we have the + # right ref. + best_match = valid_refs.max_by(&:length) + + # Partition the string into the ref and the path, ignoring the empty first value + id.partition(best_match)[1..] + end + + def use_first_path_segment?(ref) + return false unless repository_container + return false if repository_container.repository.has_ambiguous_refs? + + repository_container.repository.branch_names_include?(ref) || + repository_container.repository.tag_names_include?(ref) + end + + def extract_id_from_params + id = [params[:id] || params[:ref]] + id << ("/#{params[:path]}") unless params[:path].blank? + id.join + end + + def ref_names + return [] unless repository_container + + @ref_names ||= repository_container.repository.ref_names + end + end +end diff --git a/lib/feature.rb b/lib/feature.rb index cee6f633e78..7df692ec552 100644 --- a/lib/feature.rb +++ b/lib/feature.rb @@ -30,6 +30,20 @@ module Feature superclass.table_name = 'feature_gates' end + # Generates the same flipper_id when in a request + # If not in a request, it generates a unique flipper_id every time + class FlipperRequest + def id + Gitlab::SafeRequestStore.fetch("flipper_request_id") do + SecureRandom.uuid + end + end + + def flipper_id + "FlipperRequest:#{id}" + end + end + # To enable EE overrides class ActiveSupportCacheStoreAdapter < Flipper::Adapters::ActiveSupportCacheStore end @@ -189,7 +203,7 @@ module Feature @flipper = nil end - # This method is called from config/initializers/flipper.rb and can be used + # This method is called from config/initializers/0_inject_feature_flags.rb and can be used # to register Flipper groups. # See https://docs.gitlab.com/ee/development/feature_flags/index.html # @@ -206,6 +220,14 @@ module Feature Feature::Definition.register_hot_reloader! end + def current_request + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore[:flipper_request] ||= FlipperRequest.new + else + @flipper_request ||= FlipperRequest.new + end + end + def logger @logger ||= Feature::Logger.build end diff --git a/lib/feature/definition.rb b/lib/feature/definition.rb index 2bad7cfd33d..af60fb95c53 100644 --- a/lib/feature/definition.rb +++ b/lib/feature/definition.rb @@ -7,7 +7,7 @@ module Feature attr_reader :path attr_reader :attributes - VALID_FEATURE_NAME = %r{^#{Gitlab::Regex.sep_by_1('_', /[a-z0-9]+/)}$}.freeze + VALID_FEATURE_NAME = %r{^#{Gitlab::Regex.sep_by_1('_', /[a-z0-9]+/)}$} PARAMS.each do |param| define_method(param) do diff --git a/lib/generators/batched_background_migration/templates/batched_background_migration_dictionary.template b/lib/generators/batched_background_migration/templates/batched_background_migration_dictionary.template index 8aa08e15f48..e73bdda64eb 100644 --- a/lib/generators/batched_background_migration/templates/batched_background_migration_dictionary.template +++ b/lib/generators/batched_background_migration/templates/batched_background_migration_dictionary.template @@ -4,3 +4,7 @@ description: # Please capture what <%= class_name %> does feature_category: <%= feature_category %> introduced_by_url: # URL of the MR (or issue/commit) that introduced the migration milestone: <%= current_milestone %> +queued_migration_version: <%= migration_number %> +# Replace with the approximate date you think it's best to ensure the completion of this BBM. +finalize_after: # yyyy-mm-dd +finalized_by: # version of the migration that ensured this bbm diff --git a/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template b/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template index 502edf2c1d7..886a3bd3116 100644 --- a/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template +++ b/lib/generators/batched_background_migration/templates/queue_batched_background_migration.template @@ -17,6 +17,7 @@ class <%= migration_class_name %> < Gitlab::Database::Migration[<%= Gitlab::Data :<%= table_name %>, :<%= column_name %>, job_interval: DELAY_INTERVAL, + queued_migration_version: '<%= migration_number %>', batch_size: BATCH_SIZE, sub_batch_size: SUB_BATCH_SIZE ) diff --git a/lib/generators/gitlab/analytics/internal_events_generator.rb b/lib/generators/gitlab/analytics/internal_events_generator.rb index 083d5c31c9b..e0add9ca41d 100644 --- a/lib/generators/gitlab/analytics/internal_events_generator.rb +++ b/lib/generators/gitlab/analytics/internal_events_generator.rb @@ -6,6 +6,7 @@ module Gitlab module Analytics class InternalEventsGenerator < Rails::Generators::Base TIME_FRAME_DIRS = { + 'all' => 'counts_all', '7d' => 'counts_7d', '28d' => 'counts_28d' }.freeze @@ -81,7 +82,7 @@ module Gitlab desc: 'Name of the event that this metric counts' class_option :unique, type: :string, - optional: false, + optional: true, desc: 'Name of the event property that this metric counts' def create_metric_file @@ -140,6 +141,12 @@ module Gitlab options[:event] end + def unique(time_frame) + return if time_frame == 'all' + + "\n unique: #{options.fetch(:unique)}" + end + def ask_description(entity, type, considerations) say("") desc = ask(format(DESCRIPTION_INQUIRY, entity: entity, entity_type: type, considerations: considerations)) @@ -171,12 +178,16 @@ module Gitlab Gitlab::VERSION.match('(\d+\.\d+)').captures.first end - def class_name - 'RedisHLLMetric' + def class_name(time_frame) + time_frame == 'all' ? 'TotalCountMetric' : 'RedisHLLMetric' end def key_path(time_frame) - "count_distinct_#{options[:unique].sub('.', '_')}_from_#{event}_#{time_frame}" + if time_frame == 'all' + "count_total_#{event}" + else + "count_distinct_#{options[:unique].sub('.', '_')}_from_#{event}_#{time_frame}" + end end def metric_file_path(time_frame) @@ -188,12 +199,15 @@ module Gitlab def validate! validate_tiers! - %i[unique event mr section stage group].each do |option| + %i[event mr section stage group].each do |option| raise "The option: --#{option} is missing" unless options.key? option end time_frames.each do |time_frame| validate_time_frame!(time_frame) + + raise "The option: --unique is missing" if time_frame != 'all' && !options.key?('unique') + validate_key_path!(time_frame) end end @@ -251,7 +265,7 @@ module Gitlab end def metric_definitions - @definitions ||= Gitlab::Usage::MetricDefinition.definitions(skip_validation: true) + @definitions ||= Gitlab::Usage::MetricDefinition.definitions end def metric_definition_exists?(time_frame) diff --git a/lib/generators/gitlab/usage_metric_definition_generator.rb b/lib/generators/gitlab/usage_metric_definition_generator.rb index 5fe0ab1364d..d57a6b0b724 100644 --- a/lib/generators/gitlab/usage_metric_definition_generator.rb +++ b/lib/generators/gitlab/usage_metric_definition_generator.rb @@ -112,7 +112,7 @@ module Gitlab end def metric_definitions - @definitions ||= Gitlab::Usage::MetricDefinition.definitions(skip_validation: true) + @definitions ||= Gitlab::Usage::MetricDefinition.definitions end def metric_definition_exists?(key_path) diff --git a/lib/gitaly/server.rb b/lib/gitaly/server.rb index a816dd89e9c..38bb1f649c9 100644 --- a/lib/gitaly/server.rb +++ b/lib/gitaly/server.rb @@ -2,7 +2,7 @@ module Gitaly class Server - SHA_VERSION_REGEX = /\A\d+\.\d+\.\d+-\d+-g([a-f0-9]{8})\z/.freeze + SHA_VERSION_REGEX = /\A\d+\.\d+\.\d+-\d+-g([a-f0-9]{8})\z/ DEFAULT_REPLICATION_FACTOR = 1 class << self diff --git a/lib/gitlab.rb b/lib/gitlab.rb index d4cd62a9c21..0875b14f7d0 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -44,7 +44,7 @@ module Gitlab end end - APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))}.freeze + APP_DIRS_PATTERN = %r{^/?(app|config|ee|lib|spec|\(\w*\))} VERSION = File.read(root.join("VERSION")).strip.freeze INSTALLATION_TYPE = File.read(root.join("INSTALLATION_TYPE")).strip.freeze HTTP_PROXY_ENV_VARS = %w[http_proxy https_proxy HTTP_PROXY HTTPS_PROXY].freeze diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb index 41f94e79f91..fc0e4ab5a0d 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/base_query_builder.rb @@ -50,8 +50,7 @@ module Gitlab def filter_author(query) return query if params[:author_username].blank? - user = User.by_username(params[:author_username]).first - + user = find_user(params[:author_username]) return query.none if user.blank? query.authored(user) @@ -60,11 +59,7 @@ module Gitlab def filter_milestone_ids(query) return query if params[:milestone_title].blank? - milestone = MilestonesFinder - .new(group_ids: root_ancestor.self_and_descendant_ids, project_ids: root_ancestor.all_projects.select(:id), title: params[:milestone_title]) - .execute - .first - + milestone = find_milestone(params[:milestone_title]) return query.none if milestone.blank? query.with_milestone_id(milestone.id) @@ -115,6 +110,17 @@ module Gitlab private attr_reader :stage, :params, :root_ancestor, :stage_event_model + + def find_milestone(title) + MilestonesFinder + .new(group_ids: root_ancestor.self_and_descendant_ids, project_ids: root_ancestor.all_projects.select(:id), title: title) + .execute + .first + end + + def find_user(username) + User.by_username(username).first + end end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index 2c4b0215307..cea25ba2db4 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -12,6 +12,24 @@ module Gitlab MAX_RANGE_DAYS = 180.days.freeze DEFAULT_DATE_RANGE = 29.days # 30 including Date.today + NEGATABLE_PARAMS = [ + :assignee_username, + :author_username, + :epic_id, + :iteration_id, + :label_name, + :milestone_title, + :my_reaction_emoji, + :weight + ].freeze + + LICENSED_PARAMS = [ + :weight, + :epic_id, + :my_reaction_emoji, + :iteration_id + ].freeze + STRONG_PARAMS_DEFINITION = [ :created_before, :created_after, @@ -22,9 +40,11 @@ module Gitlab :page, :stage_id, :end_event_filter, + *LICENSED_PARAMS, label_name: [].freeze, assignee_username: [].freeze, - project_ids: [].freeze + project_ids: [].freeze, + not: NEGATABLE_PARAMS ].freeze FINDER_PARAM_NAMES = [ @@ -46,6 +66,11 @@ module Gitlab attribute :page attribute :stage_id attribute :end_event_filter + attribute :weight + attribute :epic_id + attribute :my_reaction_emoji + attribute :iteration_id + attribute :not, default: -> { {} } FINDER_PARAM_NAMES.each do |param_name| attribute param_name diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 8d7712951e1..bf3f5b61825 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -58,7 +58,8 @@ module Gitlab fetch_google_ip_list: { threshold: 10, interval: 1.minute }, project_fork_sync: { threshold: 10, interval: 30.minutes }, ai_action: { threshold: 160, interval: 8.hours }, - jobs_index: { threshold: 600, interval: 1.minute }, + vertex_embeddings_api: { threshold: 450, interval: 1.minute }, + jobs_index: { threshold: -> { application_settings.project_jobs_api_rate_limit }, interval: 1.minute }, bulk_import: { threshold: 6, interval: 1.minute }, projects_api_rate_limit_unauthenticated: { threshold: -> { application_settings.projects_api_rate_limit_unauthenticated }, interval: 10.minutes diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 9ddfc995535..fc1f7a1583c 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -426,15 +426,21 @@ module Gitlab end def unavailable_scopes_for_resource(resource) - unavailable_observability_scopes_for_resource(resource) + unavailable_observability_scopes_for_resource(resource) + + unavailable_ai_features_scopes_for_resource(resource) end def unavailable_observability_scopes_for_resource(resource) - return [] if resource.is_a?(Group) && Gitlab::Observability.enabled?(resource) + return [] if (resource.is_a?(Project) || resource.is_a?(Group)) && + Gitlab::Observability.should_enable_observability_auth_scopes?(resource) OBSERVABILITY_SCOPES end + def unavailable_ai_features_scopes_for_resource(_resource) + AI_FEATURES_SCOPES + end + def non_admin_available_scopes API_SCOPES + REPOSITORY_SCOPES + registry_scopes + OBSERVABILITY_SCOPES + AI_FEATURES_SCOPES end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index a715f17ecd6..25465e73b95 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -32,6 +32,17 @@ module Gitlab RUNNER_JOB_TOKEN_PARAM = :token PATH_DEPENDENT_FEED_TOKEN_REGEX = /\A#{User::FEED_TOKEN_PREFIX}(\h{64})-(\d+)\z/ + PARAM_TOKEN_KEYS = [ + PRIVATE_TOKEN_PARAM, + JOB_TOKEN_PARAM, + RUNNER_JOB_TOKEN_PARAM + ].map(&:to_s).freeze + HEADER_TOKEN_KEYS = [ + PRIVATE_TOKEN_HEADER, + JOB_TOKEN_HEADER, + DEPLOY_TOKEN_HEADER + ].freeze + # Check the Rails session for valid authentication details def find_user_from_warden current_request.env['warden']&.authenticate if verified_request? @@ -204,6 +215,12 @@ module Gitlab end end + def authentication_token_present? + PARAM_TOKEN_KEYS.intersection(current_request.params.keys).any? || + HEADER_TOKEN_KEYS.intersection(current_request.env.keys).any? || + parsed_oauth_token.present? + end + private def find_user_from_job_bearer_token diff --git a/lib/gitlab/auth/ldap/auth_hash.rb b/lib/gitlab/auth/ldap/auth_hash.rb index 5435355f136..6d1d1519fc2 100644 --- a/lib/gitlab/auth/ldap/auth_hash.rb +++ b/lib/gitlab/auth/ldap/auth_hash.rb @@ -6,6 +6,8 @@ module Gitlab module Auth module Ldap class AuthHash < Gitlab::Auth::OAuth::AuthHash + extend ::Gitlab::Utils::Override + def uid @uid ||= Gitlab::Auth::Ldap::Person.normalize_dn(super) end @@ -44,6 +46,12 @@ module Gitlab def ldap_config @ldap_config ||= Gitlab::Auth::Ldap::Config.new(self.provider) end + + # Overrding this method as LDAP allows email as the username ! + override :get_username + def get_username + username_claims.map { |claim| get_from_auth_hash_or_info(claim) }.find(&:presence) + end end end end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index ed7caf84558..15e8cb04ea4 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -94,7 +94,7 @@ module Gitlab def omniauth_options opts = base_options.merge( base: base, - encryption: options['encryption'], + encryption: encryption, filter: omniauth_user_filter, name_proc: name_proc, disable_verify_certificates: !options['verify_certificates'], @@ -188,6 +188,10 @@ module Gitlab options['sync_name'] end + def encryption + options['encryption'] || 'plain' + end + def name_proc if allow_username_or_email_login proc { |name| name.gsub(/@.*\z/, '') } @@ -235,7 +239,7 @@ module Gitlab end def translate_method - NET_LDAP_ENCRYPTION_METHOD[options['encryption']&.to_sym] + NET_LDAP_ENCRYPTION_METHOD[encryption.to_sym] end def tls_options diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index cce08750296..c2b49c1c068 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -68,7 +68,7 @@ module Gitlab end def provider_config - Gitlab::Auth::OAuth::Provider.config_for(@provider) || {} + Gitlab::Auth::OAuth::Provider.config_for(provider) || {} end def provider_args @@ -96,7 +96,10 @@ module Gitlab end def get_username - username_claims.map { |claim| get_from_auth_hash_or_info(claim) }.find { |name| name.presence } + username_claims.map { |claim| get_from_auth_hash_or_info(claim) } + .find { |name| name.presence } + &.split("@") + &.first end def username_and_email diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 3981594478d..d70c788dac8 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -225,7 +225,7 @@ module Gitlab if creating_linked_ldap_user? username = ldap_person.username.presence name = ldap_person.name.presence - email = ldap_person.email.first.presence + email = ldap_person.email&.first.presence end username ||= auth_hash.username @@ -272,7 +272,7 @@ module Gitlab if creating_linked_ldap_user? metadata.set_attribute_synced(:name, true) if gl_user.name == ldap_person.name - metadata.set_attribute_synced(:email, true) if gl_user.email == ldap_person.email.first + metadata.set_attribute_synced(:email, true) if gl_user.email == ldap_person.email&.first metadata.provider = ldap_person.provider end end diff --git a/lib/gitlab/background_migration/backfill_finding_id_in_vulnerabilities.rb b/lib/gitlab/background_migration/backfill_finding_id_in_vulnerabilities.rb new file mode 100644 index 00000000000..e3b77e3c7cd --- /dev/null +++ b/lib/gitlab/background_migration/backfill_finding_id_in_vulnerabilities.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills vulnerabilities.finding_id column based on vulnerability_occurrences.vulnerability_id column + class BackfillFindingIdInVulnerabilities < BatchedMigrationJob + operation_name :backfill_finding_id_in_vulnerabilities_table + scope_to ->(relation) { relation.where(finding_id: nil) } + feature_category :vulnerability_management + + class VulnerabilitiesFindings < ApplicationRecord # rubocop:disable Style/Documentation + self.table_name = "vulnerability_occurrences" + end + + def perform + each_sub_batch do |sub_batch| + connection.execute <<~SQL + UPDATE vulnerabilities + SET finding_id = vulnerability_occurrences.id + FROM vulnerability_occurrences + WHERE vulnerabilities.id IN (#{sub_batch.select(:id).to_sql}) + AND vulnerabilities.id = vulnerability_occurrences.vulnerability_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb new file mode 100644 index 00000000000..83acd8a27f7 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_has_remediations_of_vulnerability_reads.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills has_remediations column for vulnerability_reads table. + class BackfillHasRemediationsOfVulnerabilityReads < BatchedMigrationJob + operation_name :set_has_remediations + feature_category :database + + UPDATE_SQL = <<~SQL + UPDATE + vulnerability_reads + SET + has_remediations = true + FROM + (%<subquery>s) as sub_query + WHERE + vulnerability_reads.vulnerability_id = sub_query.vulnerability_id + SQL + + def perform + each_sub_batch do |sub_batch| + update_query = update_query_for(sub_batch) + + connection.execute(update_query) + end + end + + private + + def update_query_for(sub_batch) + subquery = sub_batch.joins(" + INNER JOIN vulnerability_occurrences ON + vulnerability_reads.vulnerability_id = vulnerability_occurrences.vulnerability_id") + .select("vulnerability_reads.vulnerability_id, vulnerability_occurrences.id") + .joins("INNER JOIN vulnerability_findings_remediations ON + vulnerability_occurrences.id = vulnerability_findings_remediations.vulnerability_occurrence_id") + + format(UPDATE_SQL, subquery: subquery.to_sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb index 878f89a8b3d..c8e6841c2ae 100644 --- a/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb +++ b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb @@ -10,14 +10,14 @@ module Gitlab # - https://docs.drone.io/pipeline/environment/reference/drone-system-hostname/ 'Integrations::DroneCi' => [ :drone_url, - /\Acloud\.drone\.io\z/i.freeze + /\Acloud\.drone\.io\z/i ], # This matches the logic in `Integrations::Teamcity#url_is_saas?` # - https://gitlab.com/gitlab-org/gitlab/blob/65b7fc1ad1ad33247890324e9a3396993b7718a1/app/models/integrations/teamcity.rb#L117-122 # - https://www.jetbrains.com/help/teamcity/cloud/migrate-from-teamcity-on-premises-to-teamcity-cloud.html#Migration+Process 'Integrations::Teamcity' => [ :teamcity_url, - /\A[^\.]+\.teamcity\.com\z/i.freeze + /\A[^\.]+\.teamcity\.com\z/i ] # Other CI integrations which don't seem to have a SaaS offering: diff --git a/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules2.rb b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules2.rb new file mode 100644 index 00000000000..a882b61c67d --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphans_approval_merge_request_rules2.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Deletes orphans records whenever report_type equals to scan_finding (4) or license_scanning (2) + class DeleteOrphansApprovalMergeRequestRules2 < BatchedMigrationJob + LICENSE_SCANNING_REPORT_TYPE = 2 + SCAN_FINDING_REPORT_TYPE = 4 + + scope_to ->(relation) { + relation.where(report_type: [LICENSE_SCANNING_REPORT_TYPE, SCAN_FINDING_REPORT_TYPE], + security_orchestration_policy_configuration_id: nil) + } + + operation_name :delete_all + feature_category :database + + # rubocop: disable Style/Documentation + class ApprovalMergeRequestRuleSource < ::ApplicationRecord + # rubocop: enable Style/Documentation + + self.table_name = 'approval_merge_request_rule_sources' + end + + def perform + each_sub_batch do |sub_batch| + ApprovalMergeRequestRuleSource + .where(approval_merge_request_rule_id: sub_batch.distinct.select(:id)) + .delete_all + + sub_batch.delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_orphans_approval_project_rules2.rb b/lib/gitlab/background_migration/delete_orphans_approval_project_rules2.rb new file mode 100644 index 00000000000..f6b7e184811 --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphans_approval_project_rules2.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Deletes orphans records whenever report_type equals to scan_finding (4) or license_scanning (2) + # rubocop: disable CodeReuse/ActiveRecord + class DeleteOrphansApprovalProjectRules2 < BatchedMigrationJob + LICENSE_SCANNING_REPORT_TYPE = 2 + SCAN_FINDING_REPORT_TYPE = 4 + + scope_to ->(relation) { + relation.where(report_type: [LICENSE_SCANNING_REPORT_TYPE, SCAN_FINDING_REPORT_TYPE], + security_orchestration_policy_configuration_id: nil) + } + + operation_name :delete_all + feature_category :database + + # rubocop: disable Style/Documentation + class ApprovalMergeRequestRuleSource < ::ApplicationRecord + # rubocop: enable Style/Documentation + + self.table_name = 'approval_merge_request_rule_sources' + end + + def perform + each_sub_batch do |sub_batch| + ApprovalMergeRequestRuleSource + .where(approval_project_rule_id: sub_batch.distinct.select(:id)) + .delete_all + + sub_batch.delete_all + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + end +end diff --git a/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb deleted file mode 100644 index f53f2e8ee79..00000000000 --- a/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # migrates pages from legacy storage to zip format - # we intentionally use application code here because - # it has a lot of dependencies including models, carrierwave uploaders and service objects - # and copying all or part of this code in the background migration doesn't add much value - # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54578 for discussion - class MigratePagesToZipStorage - def perform(start_id, stop_id) - # no-op - end - end - end -end diff --git a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb index 6d59a5c8651..de3c52719c3 100644 --- a/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb +++ b/lib/gitlab/background_migration/update_jira_tracker_data_deployment_type_based_on_url.rb @@ -20,7 +20,7 @@ module Gitlab # rubocop: enable Gitlab/NamespacedClass # https://rubular.com/r/uwgK7k9KH23efa - JIRA_CLOUD_REGEX = %r{^https?://[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?\.atlassian\.net$}ix.freeze + JIRA_CLOUD_REGEX = %r{^https?://[A-Za-z0-9](?:[A-Za-z0-9\-]{0,61}[A-Za-z0-9])?\.atlassian\.net$}ix def perform cloud = [] diff --git a/lib/gitlab/background_migration/update_workspaces_config_version.rb b/lib/gitlab/background_migration/update_workspaces_config_version.rb new file mode 100644 index 00000000000..77a7fc1bcca --- /dev/null +++ b/lib/gitlab/background_migration/update_workspaces_config_version.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # No op on ce + class UpdateWorkspacesConfigVersion < BatchedMigrationJob + feature_category :remote_development + def perform; end + end + end +end + +Gitlab::BackgroundMigration::UpdateWorkspacesConfigVersion.prepend_mod_with('Gitlab::BackgroundMigration::UpdateWorkspacesConfigVersion') # rubocop:disable Layout/LineLength diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb index c8520993b8e..91994c2fa95 100644 --- a/lib/gitlab/base_doorkeeper_controller.rb +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -3,8 +3,7 @@ # This is a base controller for doorkeeper. # It adds the `can?` helper used in the views. module Gitlab - # rubocop:disable Rails/ApplicationController - class BaseDoorkeeperController < ActionController::Base + class BaseDoorkeeperController < BaseActionController include Gitlab::Allowable include EnforcesTwoFactorAuthentication include SessionsHelper @@ -13,5 +12,4 @@ module Gitlab helper_method :can? end - # rubocop:enable Rails/ApplicationController end diff --git a/lib/gitlab/bitbucket_import/error_tracking.rb b/lib/gitlab/bitbucket_import/error_tracking.rb new file mode 100644 index 00000000000..eaffe34daf8 --- /dev/null +++ b/lib/gitlab/bitbucket_import/error_tracking.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module ErrorTracking + def track_import_failure!(project, exception:, **args) + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: self.class.name, + exception: exception, + **args + ) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 7f228c19b6b..9f87bb2347c 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -16,6 +16,7 @@ module Gitlab @project = project @client = Bitbucket::Client.new(project.import_data.credentials) @formatter = Gitlab::ImportFormatter.new + @ref_converter = Gitlab::BitbucketImport::RefConverter.new(project) @labels = {} @errors = [] @users = {} @@ -31,6 +32,26 @@ module Gitlab true end + def create_labels + LABELS.each do |label_params| + label = ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true) + if label.valid? + @labels[label_params[:title]] = label + else + raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.full_name}\"" + end + end + end + + def import_pull_request_comments(pull_request, merge_request) + comments = client.pull_request_comments(repo, pull_request.iid) + + inline_comments, pr_comments = comments.partition(&:inline?) + + import_inline_comments(inline_comments, pull_request, merge_request) + import_standalone_pr_comments(pr_comments, merge_request) + end + private def already_imported?(collection, iid) @@ -166,7 +187,7 @@ module Gitlab note = '' note += @formatter.author_line(comment.author) unless find_user_id(comment.author) - note += comment.note + note += @ref_converter.convert_note(comment.note.to_s) begin gitlab_issue.notes.create!( @@ -182,17 +203,6 @@ module Gitlab end end - def create_labels - LABELS.each do |label_params| - label = ::Labels::FindOrCreateService.new(nil, project, label_params).execute(skip_authorization: true) - if label.valid? - @labels[label_params[:title]] = label - else - raise "Failed to create label \"#{label_params[:title]}\" for project \"#{project.full_name}\"" - end - end - end - def import_pull_requests pull_requests = client.pull_requests(repo) @@ -242,15 +252,6 @@ module Gitlab store_pull_request_error(pull_request, e) end - def import_pull_request_comments(pull_request, merge_request) - comments = client.pull_request_comments(repo, pull_request.iid) - - inline_comments, pr_comments = comments.partition(&:inline?) - - import_inline_comments(inline_comments, pull_request, merge_request) - import_standalone_pr_comments(pr_comments, merge_request) - end - def import_inline_comments(inline_comments, pull_request, merge_request) position_map = {} discussion_map = {} @@ -319,8 +320,7 @@ module Gitlab def comment_note(comment) author = @formatter.author_line(comment.author) unless find_user_id(comment.author) - - author.to_s + comment.note.to_s + author.to_s + @ref_converter.convert_note(comment.note.to_s) end def log_base_data diff --git a/lib/gitlab/bitbucket_import/importers/issue_importer.rb b/lib/gitlab/bitbucket_import/importers/issue_importer.rb new file mode 100644 index 00000000000..2c3be67eabc --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/issue_importer.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class IssueImporter + include Loggable + include ErrorTracking + + def initialize(project, hash) + @project = project + @formatter = Gitlab::ImportFormatter.new + @user_finder = UserFinder.new(project) + @object = hash.with_indifferent_access + end + + def execute + log_info(import_stage: 'import_issue', message: 'starting', iid: object[:iid]) + + description = '' + description += author_line + description += object[:description] if object[:description] + + milestone = object[:milestone] ? project.milestones.find_or_create_by(title: object[:milestone]) : nil # rubocop: disable CodeReuse/ActiveRecord + + attributes = { + iid: object[:iid], + title: object[:title], + description: description, + state_id: Issue.available_states[object[:state]], + author_id: author_id, + assignee_ids: [author_id], + namespace_id: project.project_namespace_id, + milestone: milestone, + work_item_type_id: object[:issue_type_id], + label_ids: [object[:label_id]].compact, + created_at: object[:created_at], + updated_at: object[:updated_at] + } + + project.issues.create!(attributes) + + log_info(import_stage: 'import_issue', message: 'finished', iid: object[:iid]) + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + attr_reader :object, :project, :formatter, :user_finder + + def author_line + return '' if find_user_id + + formatter.author_line(object[:author]) + end + + def find_user_id + user_finder.find_user_id(object[:author]) + end + + def author_id + user_finder.gitlab_user_id(project, object[:author]) + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/issue_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/issue_notes_importer.rb new file mode 100644 index 00000000000..ac0e039939f --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/issue_notes_importer.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class IssueNotesImporter + include ParallelScheduling + + def initialize(project, hash) + @project = project + @formatter = Gitlab::ImportFormatter.new + @user_finder = UserFinder.new(project) + @ref_converter = Gitlab::BitbucketImport::RefConverter.new(project) + @object = hash.with_indifferent_access + end + + def execute + log_info(import_stage: 'import_issue_notes', message: 'starting', iid: object[:iid]) + + issue = project.issues.find_by(iid: object[:iid]) # rubocop: disable CodeReuse/ActiveRecord + + if issue + client.issue_comments(project.import_source, issue.iid).each do |comment| + next unless comment.note.present? + + note = '' + note += formatter.author_line(comment.author) unless user_finder.find_user_id(comment.author) + note += ref_converter.convert_note(comment.note) + + issue.notes.create!( + project: project, + note: note, + author_id: user_finder.gitlab_user_id(project, comment.author), + created_at: comment.created_at, + updated_at: comment.updated_at + ) + end + end + + log_info(import_stage: 'import_issue_notes', message: 'finished', iid: object[:iid]) + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + attr_reader :object, :project, :formatter, :user_finder, :ref_converter + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/issues_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_importer.rb new file mode 100644 index 00000000000..6162433e701 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/issues_importer.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class IssuesImporter + include ParallelScheduling + + def execute + log_info(import_stage: 'import_issues', message: 'importing issues') + + issues = client.issues(project.import_source) + + labels = build_labels_hash + + issues.each do |issue| + job_waiter.jobs_remaining += 1 + + next if already_enqueued?(issue) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + issue_hash = issue.to_hash.merge({ issue_type_id: default_issue_type_id, label_id: labels[issue.kind] }) + sidekiq_worker_class.perform_in(job_delay, project.id, issue_hash, job_waiter.key) + + mark_as_enqueued(issue) + end + + job_waiter + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + def sidekiq_worker_class + ImportIssueWorker + end + + def collection_method + :issues + end + + def id_for_already_enqueued_cache(object) + object.iid + end + + def default_issue_type_id + ::WorkItems::Type.default_issue_type.id + end + + def build_labels_hash + labels = {} + project.labels.each { |l| labels[l.title.to_s] = l.id } + labels + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb new file mode 100644 index 00000000000..03dcc645f07 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/issues_notes_importer.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class IssuesNotesImporter + include ParallelScheduling + + def execute + project.issues.find_each do |issue| + job_waiter.jobs_remaining += 1 + + next if already_enqueued?(issue) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + sidekiq_class.perform_in(job_delay, project.id, { iid: issue.iid }, job_waiter.key) + + mark_as_enqueued(issue) + end + + job_waiter + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + attr_reader :project + + def sidekiq_class + ImportIssueNotesWorker + end + + def id_for_already_enqueued_cache(object) + object.iid + end + + def collection_method + :issues_notes + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/lfs_object_importer.rb b/lib/gitlab/bitbucket_import/importers/lfs_object_importer.rb new file mode 100644 index 00000000000..06b30c7b496 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/lfs_object_importer.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class LfsObjectImporter + include Loggable + include ErrorTracking + + def initialize(project, lfs_attributes) + @project = project + @lfs_download_object = LfsDownloadObject.new(**lfs_attributes.symbolize_keys) + end + + def execute + log_info(import_stage: 'import_lfs_object', message: 'starting', oid: lfs_download_object.oid) + + lfs_download_object.validate! + Projects::LfsPointers::LfsDownloadService.new(project, lfs_download_object).execute + + log_info(import_stage: 'import_lfs_object', message: 'finished', oid: lfs_download_object.oid) + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + attr_reader :lfs_download_object, :project + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/lfs_objects_importer.rb b/lib/gitlab/bitbucket_import/importers/lfs_objects_importer.rb new file mode 100644 index 00000000000..aa9ff7000f1 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/lfs_objects_importer.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class LfsObjectsImporter + include ParallelScheduling + + def execute + log_info(import_stage: 'import_lfs_objects', message: 'starting') + + download_service = Projects::LfsPointers::LfsObjectDownloadListService.new(project) + + begin + queue_workers(download_service) if project.lfs_enabled? + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + log_info(import_stage: 'import_lfs_objects', message: 'finished') + + job_waiter + end + + private + + def sidekiq_worker_class + ImportLfsObjectWorker + end + + def collection_method + :lfs_objects + end + + def id_for_already_enqueued_cache(object) + object.oid + end + + def queue_workers(download_service) + download_service.each_list_item do |lfs_download_object| + # Needs to come before `already_enqueued?` as `jobs_remaining` resets to zero when the job restarts and + # jobs_remaining needs to be the total amount of enqueued jobs + job_waiter.jobs_remaining += 1 + + next if already_enqueued?(lfs_download_object) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + sidekiq_worker_class.perform_in(job_delay, project.id, lfs_download_object.to_hash, job_waiter.key) + + mark_as_enqueued(lfs_download_object) + end + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb index d76e08e1039..a18d50e8fce 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb @@ -5,6 +5,7 @@ module Gitlab module Importers class PullRequestImporter include Loggable + include ErrorTracking def initialize(project, hash) @project = project @@ -48,7 +49,7 @@ module Gitlab log_info(import_stage: 'import_pull_request', message: 'finished', iid: object[:iid]) rescue StandardError => e - Gitlab::Import::ImportFailureService.track(project_id: project.id, exception: e) + track_import_failure!(project, exception: e) end private diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb new file mode 100644 index 00000000000..8ea8b1562f2 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class PullRequestNotesImporter + include Loggable + include ErrorTracking + + def initialize(project, hash) + @project = project + @importer = Gitlab::BitbucketImport::Importer.new(project) + @object = hash.with_indifferent_access + end + + def execute + log_info(import_stage: 'import_pull_request_notes', message: 'starting', iid: object[:iid]) + + merge_request = project.merge_requests.find_by(iid: object[:iid]) # rubocop: disable CodeReuse/ActiveRecord + + importer.import_pull_request_comments(merge_request, merge_request) if merge_request + + log_info(import_stage: 'import_pull_request_notes', message: 'finished', iid: object[:iid]) + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + attr_reader :object, :project, :importer + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb new file mode 100644 index 00000000000..a1b0c2a5afe --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/pull_requests_notes_importer.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class PullRequestsNotesImporter + include ParallelScheduling + + def execute + project.merge_requests.find_each do |merge_request| + job_waiter.jobs_remaining += 1 + + next if already_enqueued?(merge_request) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + sidekiq_worker_class.perform_in(job_delay, project.id, { iid: merge_request.iid }, job_waiter.key) + + mark_as_enqueued(merge_request) + end + + job_waiter + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + attr_reader :project + + def sidekiq_worker_class + ImportPullRequestNotesWorker + end + + def id_for_already_enqueued_cache(object) + object.iid + end + + def collection_method + :merge_requests_notes + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_import/importers/repository_importer.rb index 7b0362b6ec6..b8c0ba69d37 100644 --- a/lib/gitlab/bitbucket_import/importers/repository_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/repository_importer.rb @@ -23,6 +23,7 @@ module Gitlab end import_wiki + create_labels log_info(import_stage: 'import_repository', message: 'finished import') @@ -59,6 +60,11 @@ module Gitlab ) end + def create_labels + importer = Gitlab::BitbucketImport::Importer.new(project) + importer.create_labels + end + def wiki WikiFormatter.new(project) end diff --git a/lib/gitlab/bitbucket_import/parallel_scheduling.rb b/lib/gitlab/bitbucket_import/parallel_scheduling.rb index f4df9a35526..ca2597ea5cf 100644 --- a/lib/gitlab/bitbucket_import/parallel_scheduling.rb +++ b/lib/gitlab/bitbucket_import/parallel_scheduling.rb @@ -4,6 +4,7 @@ module Gitlab module BitbucketImport module ParallelScheduling include Loggable + include ErrorTracking attr_reader :project, :already_enqueued_cache_key, :job_waiter_cache_key @@ -79,15 +80,6 @@ module Gitlab (multiplier * 1.minute) + 1.second end - - def track_import_failure!(project, exception:, **args) - Gitlab::Import::ImportFailureService.track( - project_id: project.id, - error_source: self.class.name, - exception: exception, - **args - ) - end end end end diff --git a/lib/gitlab/bitbucket_import/ref_converter.rb b/lib/gitlab/bitbucket_import/ref_converter.rb new file mode 100644 index 00000000000..1159159a76d --- /dev/null +++ b/lib/gitlab/bitbucket_import/ref_converter.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + class RefConverter + REPO_MATCHER = 'https://bitbucket.org/%s' + PR_NOTE_ISSUE_NAME_REGEX = '(?<=/)[^/\)]+(?=\)[^/]*$)' + UNWANTED_NOTE_REF_HTML = "{: data-inline-card='' }" + + attr_reader :project + + def initialize(project) + @project = project + end + + def convert_note(note) + repo_matcher = REPO_MATCHER % project.import_source + + return note unless note.match?(repo_matcher) + + note = note.gsub(repo_matcher, url_helpers.project_url(project)) + .gsub(UNWANTED_NOTE_REF_HTML, '') + .strip + + if note.match?('issues') + note.gsub!('issues', '-/issues') + note.gsub!(issue_name(note), '') + else + note.gsub!('pull-requests', '-/merge_requests') + note.gsub!('src', '-/blob') + note.gsub!('lines-', 'L') + end + + note + end + + private + + def url_helpers + Rails.application.routes.url_helpers + end + + def issue_name(note) + note.match(PR_NOTE_ISSUE_NAME_REGEX)[0] + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb index 34963452192..0d4de385f5e 100644 --- a/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/pull_request_importer.rb @@ -30,7 +30,7 @@ module Gitlab reviewer_ids: reviewers, source_project_id: project.id, source_branch: Gitlab::Git.ref_name(object[:source_branch_name]), - source_branch_sha: object[:source_branch_sha], + source_branch_sha: source_branch_sha, target_project_id: project.id, target_branch: Gitlab::Git.ref_name(object[:target_branch_name]), target_branch_sha: object[:target_branch_sha], @@ -68,6 +68,14 @@ module Gitlab end end end + + def source_branch_sha + source_branch_sha = project.repository.commit(object[:source_branch_sha])&.sha + + return source_branch_sha if source_branch_sha + + project.repository.find_commits_by_message(object[:source_branch_sha])&.first&.sha + end end end end diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb index 92ec10bf037..ae73681f7f8 100644 --- a/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb @@ -20,6 +20,22 @@ module Gitlab break if pull_requests.empty? + commits_to_fetch = pull_requests.filter_map do |pull_request| + next if already_processed?(pull_request) + next unless pull_request.merged? || pull_request.closed? + + [pull_request.source_branch_sha, pull_request.target_branch_sha] + end.flatten + + # Bitbucket Server keeps tracks of references for open pull requests in + # refs/heads/pull-requests, but closed and merged requests get moved + # into hidden internal refs under stash-refs/pull-requests. As a result, + # they are not fetched by default. + # + # This method call explicitly fetches head and start commits for affected pull requests. + # That allows us to correctly assign diffs and commits to merge requests. + fetch_missing_commits(commits_to_fetch) + pull_requests.each do |pull_request| # Needs to come before `already_processed?` as `jobs_remaining` resets to zero when the job restarts and # jobs_remaining needs to be the total amount of enqueued jobs @@ -42,6 +58,15 @@ module Gitlab private + def fetch_missing_commits(commits_to_fetch) + return if commits_to_fetch.blank? + return unless Feature.enabled?(:fetch_commits_for_bitbucket_server, project.group) + + project.repository.fetch_remote(project.import_url, refmap: commits_to_fetch, prune: false) + rescue StandardError => e + track_import_failure!(project, exception: e) + end + def sidekiq_worker_class ImportPullRequestWorker end diff --git a/lib/gitlab/bitbucket_server_import/project_creator.rb b/lib/gitlab/bitbucket_server_import/project_creator.rb index ddc678abdd8..be60e431b80 100644 --- a/lib/gitlab/bitbucket_server_import/project_creator.rb +++ b/lib/gitlab/bitbucket_server_import/project_creator.rb @@ -3,9 +3,9 @@ module Gitlab module BitbucketServerImport class ProjectCreator - attr_reader :project_key, :repo_slug, :repo, :name, :namespace, :current_user, :session_data + attr_reader :project_key, :repo_slug, :repo, :name, :namespace, :current_user, :session_data, :timeout_strategy - def initialize(project_key, repo_slug, repo, name, namespace, current_user, session_data) + def initialize(project_key, repo_slug, repo, name, namespace, current_user, session_data, timeout_strategy) @project_key = project_key @repo_slug = repo_slug @repo = repo @@ -13,6 +13,7 @@ module Gitlab @namespace = namespace @current_user = current_user @session_data = session_data + @timeout_strategy = timeout_strategy end def execute @@ -28,7 +29,7 @@ module Gitlab import_url: repo.clone_url, import_data: { credentials: session_data, - data: { project_key: project_key, repo_slug: repo_slug } + data: { project_key: project_key, repo_slug: repo_slug, timeout_strategy: timeout_strategy } }, skip_wiki: true ).execute diff --git a/lib/gitlab/changelog/generator.rb b/lib/gitlab/changelog/generator.rb index a80ca0728f9..0e546c5eb60 100644 --- a/lib/gitlab/changelog/generator.rb +++ b/lib/gitlab/changelog/generator.rb @@ -6,7 +6,7 @@ module Gitlab class Generator # The regex used to parse a release header. RELEASE_REGEX = - /^##\s+(?<version>#{Gitlab::Regex.unbounded_semver_regex})/.freeze + /^##\s+(?<version>#{Gitlab::Regex.unbounded_semver_regex})/ # The `input` argument must be a `String` containing the existing # changelog Markdown. If no changelog exists, this should be an empty diff --git a/lib/gitlab/chat.rb b/lib/gitlab/chat.rb deleted file mode 100644 index 30e9989d270..00000000000 --- a/lib/gitlab/chat.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Chat - # Returns `true` if Chatops is available for the current instance. - def self.available? - ::Feature.enabled?(:chatops) - end - end -end diff --git a/lib/gitlab/checks/global_file_size_check.rb b/lib/gitlab/checks/global_file_size_check.rb index 62facf52239..ff24467e9cc 100644 --- a/lib/gitlab/checks/global_file_size_check.rb +++ b/lib/gitlab/checks/global_file_size_check.rb @@ -17,16 +17,34 @@ module Gitlab ).find if oversized_blobs.present? + + blob_details = {} + blob_id_size_msg = "" + oversized_blobs.each do |blob| + blob_details[blob.id] = { "size" => blob.size } + + # blob size is in byte, divide it by "/ 1024.0 / 1024.0" to get MiB + blob_id_size_msg += "- #{blob.id} (#{(blob.size / 1024.0 / 1024.0).round(2)} MiB) \n" + end + + oversize_err_msg = <<~OVERSIZE_ERR_MSG + You are attempting to check in one or more blobs which exceed the #{file_size_limit}MiB limit: + + #{blob_id_size_msg} + To resolve this error, you must either reduce the size of the above blobs, or utilize LFS. + You may use "git ls-tree -r HEAD | grep $BLOB_ID" to see the file path. + Please refer to #{Rails.application.routes.url_helpers.help_page_url('user/free_push_limit')} and + #{Rails.application.routes.url_helpers.help_page_url('administration/settings/account_and_limit_settings')} + for further information. + OVERSIZE_ERR_MSG + Gitlab::AppJsonLogger.info( message: 'Found blob over global limit', - blob_sizes: oversized_blobs.map(&:size) + blob_sizes: oversized_blobs.map(&:size), + blob_details: blob_details ) - if enforce_global_file_size_limit? - raise ::Gitlab::GitAccess::ForbiddenError, - "Changes include a file that is larger than the allowed size of #{file_size_limit} MiB. " \ - "Use Git LFS to manage this file.)" - end + raise ::Gitlab::GitAccess::ForbiddenError, oversize_err_msg if enforce_global_file_size_limit? end end diff --git a/lib/gitlab/checks/security/policy_check.rb b/lib/gitlab/checks/security/policy_check.rb new file mode 100644 index 00000000000..b2be393351a --- /dev/null +++ b/lib/gitlab/checks/security/policy_check.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + module Security + class PolicyCheck < BaseSingleChecker + def validate!; end + end + end + end +end + +Gitlab::Checks::Security::PolicyCheck.prepend_mod diff --git a/lib/gitlab/checks/single_change_access.rb b/lib/gitlab/checks/single_change_access.rb index 9f427e98e55..625524cf2bc 100644 --- a/lib/gitlab/checks/single_change_access.rb +++ b/lib/gitlab/checks/single_change_access.rb @@ -54,6 +54,7 @@ module Gitlab Gitlab::Checks::PushCheck.new(self).validate! Gitlab::Checks::BranchCheck.new(self).validate! Gitlab::Checks::TagCheck.new(self).validate! + Gitlab::Checks::Security::PolicyCheck.new(self).validate! end def commits_check diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb index 4505bcb5411..d5addab74b8 100644 --- a/lib/gitlab/checks/tag_check.rb +++ b/lib/gitlab/checks/tag_check.rb @@ -11,7 +11,8 @@ module Gitlab delete_protected_tag_non_web: 'You can only delete protected tags using the web interface.', create_protected_tag: 'You are not allowed to create this tag as it is protected.', default_branch_collision: 'You cannot use default branch name to create a tag', - prohibited_tag_name: 'You cannot create a tag with a prohibited pattern.' + prohibited_tag_name: 'You cannot create a tag with a prohibited pattern.', + prohibited_tag_name_encoding: 'Tag names must be valid when converted to UTF-8 encoding' }.freeze LOG_MESSAGES = { @@ -46,6 +47,16 @@ module Gitlab if tag_name.start_with?("refs/tags/") # rubocop: disable Style/GuardClause raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name] end + + # rubocop: disable Style/GuardClause + # rubocop: disable Style/SoleNestedConditional + if Feature.enabled?(:prohibited_tag_name_encoding_check, project) + unless Gitlab::EncodingHelper.force_encode_utf8(tag_name).valid_encoding? + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_tag_name_encoding] + end + end + # rubocop: enable Style/SoleNestedConditional + # rubocop: enable Style/GuardClause end def protected_tag_checks diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index 5748b8e34cf..7d9235ac460 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -11,8 +11,8 @@ module Gitlab ParserError = Class.new(StandardError) InvalidStreamError = Class.new(StandardError) - VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/.freeze - INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)}.freeze + VERSION_PATTERN = /^[\w\s]+(\d+\.\d+\.\d+)/ + INVALID_PATH_PATTERN = %r{(^\.?\.?/)|(/\.?\.?/)} attr_reader :stream, :path, :full_version diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb index 81efbdb297b..48b138b0258 100644 --- a/lib/gitlab/ci/build/context/build.rb +++ b/lib/gitlab/ci/build/context/build.rb @@ -30,8 +30,16 @@ module Gitlab ::Ci::Build.new(build_attributes) end + # Assigning tags and needs is slow and they are not needed for rules + # evaluation since we don't use them to compute the variables at this point. def build_attributes - attributes.merge(pipeline_attributes, ci_stage_attributes) + if pipeline.reduced_build_attributes_list_for_rules? + attributes + .except(:tag_list, :needs_attributes) + .merge!(pipeline_attributes, ci_stage_attributes) + else + attributes.merge(pipeline_attributes, ci_stage_attributes) + end end def ci_stage_attributes diff --git a/lib/gitlab/ci/build/duration_parser.rb b/lib/gitlab/ci/build/duration_parser.rb index 97049a4f876..9385dccd5f3 100644 --- a/lib/gitlab/ci/build/duration_parser.rb +++ b/lib/gitlab/ci/build/duration_parser.rb @@ -41,7 +41,7 @@ module Gitlab def parse return if never? - ChronicDuration.parse(value, use_complete_matcher: true) + ChronicDuration.parse(value) end def validation_cache diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb index 17c784c4d54..551284d9099 100644 --- a/lib/gitlab/ci/components/instance_path.rb +++ b/lib/gitlab/ci/components/instance_path.rb @@ -7,19 +7,17 @@ module Gitlab include Gitlab::Utils::StrongMemoize LATEST_VERSION_KEYWORD = '~latest' - TEMPLATES_DIR = 'templates' def self.match?(address) address.include?('@') && address.start_with?(Settings.gitlab_ci['component_fqdn']) end - attr_reader :host, :project_file_path + attr_reader :host - def initialize(address:, content_filename:) + def initialize(address:) @full_path, @version = address.to_s.split('@', 2) - @content_filename = content_filename @host = Settings.gitlab_ci['component_fqdn'] - @project_file_path = nil + @component_project = ::Ci::Catalog::ComponentsProject.new(project, sha) end def fetch_content!(current_user:) @@ -28,7 +26,7 @@ module Gitlab raise Gitlab::Access::AccessDeniedError unless Ability.allowed?(current_user, :download_code, project) - content(simple_template_path) || content(complex_template_path) || content(legacy_template_path) + @component_project.fetch_component(component_name) end def project @@ -46,16 +44,7 @@ module Gitlab private - attr_reader :version, :path - - def instance_path - @full_path.delete_prefix(host) - end - - def component_path - instance_path.delete_prefix(project.full_path).delete_prefix('/') - end - strong_memoize_attr :component_path + attr_reader :version # Given a path like "my-org/sub-group/the-project/path/to/component" # find the project "my-org/sub-group/the-project" by looking at all possible paths. @@ -65,45 +54,23 @@ module Gitlab while index = path.rindex('/') # find index of last `/` in a path possible_paths << (path = path[0..index - 1]) end - # remove shortest path as it is group possible_paths.pop ::Project.where_full_path_in(possible_paths).take # rubocop: disable CodeReuse/ActiveRecord end - def latest_version_sha - project.releases.latest&.sha - end - - # A simple template consists of a single file - def simple_template_path - # Extract this line and move to fetch_content once we remove legacy fetching - return unless templates_dir_exists? && component_path.index('/').nil? - - @project_file_path = File.join(TEMPLATES_DIR, "#{component_path}.yml") - end - - # A complex template is directory-based and may consist of multiple files. - # Given a path like "my-org/sub-group/the-project/templates/component" - # returns the entry point path: "templates/component/template.yml". - def complex_template_path - # Extract this line and move to fetch_content once we remove legacy fetching - return unless templates_dir_exists? && component_path.index('/').nil? - - @project_file_path = File.join(TEMPLATES_DIR, component_path, @content_filename) - end - - def legacy_template_path - @project_file_path = File.join(component_path, @content_filename).delete_prefix('/') + def instance_path + @full_path.delete_prefix(host) end - def templates_dir_exists? - project.repository.tree.trees.map(&:name).include?(TEMPLATES_DIR) + def component_name + instance_path.delete_prefix(project.full_path).delete_prefix('/') end + strong_memoize_attr :component_name - def content(path) - project.repository.blob_data_at(sha, path) + def latest_version_sha + project.releases.latest&.sha end end end diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 27206d7e3a8..3fd07811daf 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_WHEN = %w[on_success on_failure always].freeze ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude public].freeze - EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze + EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/ EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces" attributes ALLOWED_KEYS diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index c40d665f320..bf8a99ef45e 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -177,7 +177,7 @@ module Gitlab def parsed_timeout return unless has_timeout? - ChronicDuration.parse(timeout.to_s, use_complete_matcher: true) + ChronicDuration.parse(timeout.to_s) end def ignored? diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index efba81c7420..b3c802e5657 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :location, :params, :context, :errors - YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i.freeze + YAML_WHITELIST_EXTENSION = /.+\.(yml|yaml)$/i def initialize(params, context) @params = params @@ -114,7 +114,9 @@ module Gitlab def content_result context.logger.instrument(:config_file_fetch_content_hash) do - ::Gitlab::Ci::Config::Yaml::Loader.new(content, inputs: content_inputs).load + ::Gitlab::Ci::Config::Yaml::Loader.new( + content, inputs: content_inputs, variables: context.variables + ).load end end strong_memoize_attr :content_result diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb index de6de1bb7a8..03063e76dde 100644 --- a/lib/gitlab/ci/config/external/file/component.rb +++ b/lib/gitlab/ci/config/external/file/component.rb @@ -20,7 +20,7 @@ module Gitlab ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event('cicd_component_usage', values: context.user.id) - component_result.payload.fetch(:content) + component_payload.fetch(:content) end strong_memoize_attr :content @@ -65,30 +65,30 @@ module Gitlab override :expand_context_attrs def expand_context_attrs { - project: component_path.project, - sha: component_path.sha, + project: component_payload.fetch(:project), + sha: component_payload.fetch(:sha), user: context.user, variables: context.variables } end def masked_blob - return unless component_path + return unless component_payload context.mask_variables_from( Gitlab::Routing.url_helpers.project_blob_url( - component_path.project, - ::File.join(component_path.sha, component_path.project_file_path)) + component_payload.fetch(:project), + ::File.join(component_payload.fetch(:sha), component_payload.fetch(:path))) ) end strong_memoize_attr :masked_blob - def component_path + def component_payload return unless component_result.success? - component_result.payload.fetch(:path) + component_result.payload end - strong_memoize_attr :component_path + strong_memoize_attr :component_payload end end end diff --git a/lib/gitlab/ci/config/header/input.rb b/lib/gitlab/ci/config/header/input.rb index 76a89a3080e..dcb96006459 100644 --- a/lib/gitlab/ci/config/header/input.rb +++ b/lib/gitlab/ci/config/header/input.rb @@ -11,12 +11,16 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - attributes :default, :type, prefix: :input + ALLOWED_KEYS = %i[default description regex type].freeze + + attributes ALLOWED_KEYS, prefix: :input validations do - validates :config, type: Hash, allowed_keys: [:default, :type] + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS validates :key, alphanumeric: true validates :input_default, alphanumeric: true, allow_nil: true + validates :input_description, alphanumeric: true, allow_nil: true + validates :input_regex, type: String, allow_nil: true validates :input_type, allow_nil: true, allowed_values: Interpolation::Inputs.input_types end end diff --git a/lib/gitlab/ci/config/interpolation/block.rb b/lib/gitlab/ci/config/interpolation/block.rb index cf8420f924e..aec19299e86 100644 --- a/lib/gitlab/ci/config/interpolation/block.rb +++ b/lib/gitlab/ci/config/interpolation/block.rb @@ -62,7 +62,7 @@ module Gitlab return @errors.concat(access.errors) unless access.valid? return @errors.push('too many functions in interpolation block') if functions.count > MAX_FUNCTIONS - result = Interpolation::FunctionsStack.new(functions).evaluate(access.value) + result = Interpolation::FunctionsStack.new(functions, ctx).evaluate(access.value) if result.success? @value = result.value diff --git a/lib/gitlab/ci/config/interpolation/context.rb b/lib/gitlab/ci/config/interpolation/context.rb index f5e7db03291..19ea619f7da 100644 --- a/lib/gitlab/ci/config/interpolation/context.rb +++ b/lib/gitlab/ci/config/interpolation/context.rb @@ -14,8 +14,11 @@ module Gitlab MAX_DEPTH = 3 - def initialize(hash) - @context = hash + attr_reader :variables + + def initialize(data, variables: []) + @data = data + @variables = Ci::Variables::Collection.fabricate(variables) raise ContextTooComplexError if depth > MAX_DEPTH end @@ -32,25 +35,25 @@ module Gitlab end def depth - deep_depth(@context) + deep_depth(@data) end def fetch(field) - @context.fetch(field) + @data.fetch(field) end def key?(name) - @context.key?(name) + @data.key?(name) end def to_h - @context.to_h + @data.to_h end private - def deep_depth(context, depth = 0) - values = context.values.map do |value| + def deep_depth(data, depth = 0) + values = data.values.map do |value| if value.is_a?(Hash) deep_depth(value, depth + 1) else @@ -61,10 +64,10 @@ module Gitlab values.max.to_i end - def self.fabricate(context) + def self.fabricate(context, variables: []) case context when Hash - new(context) + new(context, variables: variables) when Interpolation::Context context else diff --git a/lib/gitlab/ci/config/interpolation/functions/base.rb b/lib/gitlab/ci/config/interpolation/functions/base.rb index b9ce8cdc5bc..b04152a1558 100644 --- a/lib/gitlab/ci/config/interpolation/functions/base.rb +++ b/lib/gitlab/ci/config/interpolation/functions/base.rb @@ -20,9 +20,10 @@ module Gitlab function_expression_pattern.match?(function_expression) end - def initialize(function_expression) + def initialize(function_expression, ctx) @errors = [] @function_args = parse_args(function_expression) + @ctx = ctx end def valid? @@ -35,10 +36,11 @@ module Gitlab private - attr_reader :function_args + attr_reader :function_args, :ctx def error(message) errors << "error in `#{self.class.name}` function: #{message}" + nil end def parse_args(function_expression) diff --git a/lib/gitlab/ci/config/interpolation/functions/expand_vars.rb b/lib/gitlab/ci/config/interpolation/functions/expand_vars.rb new file mode 100644 index 00000000000..658964018b5 --- /dev/null +++ b/lib/gitlab/ci/config/interpolation/functions/expand_vars.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Interpolation + module Functions + class ExpandVars < Base + def self.function_expression_pattern + /^#{name}$/ + end + + def self.name + 'expand_vars' + end + + def execute(input_value) + unless input_value.is_a?(String) + error("invalid input type: #{self.class.name} can only be used with string inputs") + return + end + + ExpandVariables.expand_existing(input_value, ctx.variables, fail_on_masked: true) + rescue ExpandVariables::VariableExpansionError => e + error("variable expansion error: #{e.message}") + nil + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/interpolation/functions_stack.rb b/lib/gitlab/ci/config/interpolation/functions_stack.rb index 951d1121d4f..4cb3e67b3e3 100644 --- a/lib/gitlab/ci/config/interpolation/functions_stack.rb +++ b/lib/gitlab/ci/config/interpolation/functions_stack.rb @@ -16,12 +16,14 @@ module Gitlab end FUNCTIONS = [ - Functions::Truncate + Functions::Truncate, + Functions::ExpandVars ].freeze attr_reader :errors - def initialize(function_expressions) + def initialize(function_expressions, ctx) + @ctx = ctx @errors = [] @functions = build_stack(function_expressions) end @@ -48,14 +50,14 @@ module Gitlab private - attr_reader :functions + attr_reader :functions, :ctx def build_stack(function_expressions) function_expressions.map do |function_expression| matching_function = FUNCTIONS.find { |function| function.matches?(function_expression) } if matching_function.present? - matching_function.new(function_expression) + matching_function.new(function_expression, ctx) else message = "no function matching `#{function_expression}`: " \ 'check that the function name, arguments, and types are correct' diff --git a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb index 5648c4d31ea..ba519776635 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb @@ -62,7 +62,15 @@ module Gitlab end # validate provided value - error("provided value is not a #{self.class.type_name}") unless valid_value?(actual_value) + return error("provided value is not a #{self.class.type_name}") unless valid_value?(actual_value) + + validate_regex! + end + + def validate_regex! + return unless spec.key?(:regex) + + error('RegEx validation can only be used with string inputs') end def error(message) diff --git a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb index 39870582d0c..3f40e851f11 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb @@ -25,6 +25,24 @@ module Gitlab def valid_value?(value) value.nil? || value.is_a?(String) end + + private + + def validate_regex! + return unless spec.key?(:regex) + + safe_regex = ::Gitlab::UntrustedRegexp.new(spec[:regex]) + + return if safe_regex.match?(actual_value) + + if value.nil? + error('default value does not match required RegEx pattern') + else + error('provided value does not match required RegEx pattern') + end + rescue RegexpError + error('invalid regular expression') + end end end end diff --git a/lib/gitlab/ci/config/interpolation/interpolator.rb b/lib/gitlab/ci/config/interpolation/interpolator.rb index 95c419d7427..5b21b777c1d 100644 --- a/lib/gitlab/ci/config/interpolation/interpolator.rb +++ b/lib/gitlab/ci/config/interpolation/interpolator.rb @@ -8,11 +8,12 @@ module Gitlab # Performs CI config file interpolation, and surfaces all possible interpolation errors. # class Interpolator - attr_reader :config, :args, :errors + attr_reader :config, :args, :variables, :errors - def initialize(config, args) + def initialize(config, args, variables) @config = config @args = args.to_h + @variables = variables @errors = [] @interpolated = false end @@ -86,7 +87,7 @@ module Gitlab end def context - @context ||= Context.new({ inputs: inputs.to_hash }) + @context ||= Context.new({ inputs: inputs.to_hash }, variables: variables) end def template diff --git a/lib/gitlab/ci/config/yaml/loader.rb b/lib/gitlab/ci/config/yaml/loader.rb index 5d56061a8bb..1e9ac2b3dd5 100644 --- a/lib/gitlab/ci/config/yaml/loader.rb +++ b/lib/gitlab/ci/config/yaml/loader.rb @@ -10,9 +10,10 @@ module Gitlab AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze MAX_DOCUMENTS = 2 - def initialize(content, inputs: {}) + def initialize(content, inputs: {}, variables: []) @content = content @inputs = inputs + @variables = variables end def load @@ -20,7 +21,7 @@ module Gitlab return yaml_result unless yaml_result.valid? - interpolator = Interpolation::Interpolator.new(yaml_result, inputs) + interpolator = Interpolation::Interpolator.new(yaml_result, inputs, variables) interpolator.interpolate! @@ -32,16 +33,16 @@ module Gitlab end end - private - - attr_reader :content, :inputs - def load_uninterpolated_yaml Yaml::Result.new(config: load_yaml!, error: nil) rescue ::Gitlab::Config::Loader::FormatError => e Yaml::Result.new(error: e.message, error_class: e) end + private + + attr_reader :content, :inputs, :variables + def load_yaml! ensure_custom_tags diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb index a68cfde6653..0e7e9230467 100644 --- a/lib/gitlab/ci/config/yaml/result.rb +++ b/lib/gitlab/ci/config/yaml/result.rb @@ -39,6 +39,10 @@ module Gitlab @config.first || {} end + + def inputs + (has_header? && header[:spec][:inputs]) || {} + end end end end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 54861e2769e..f213bc83d90 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -25,12 +25,12 @@ module Gitlab LOG_MAX_DURATION_THRESHOLD = 2.seconds - def initialize(project:, current_user:, sha: nil) + def initialize(project:, current_user:, sha: nil, verify_project_sha: true) @project = project @current_user = current_user # If the `sha` is not provided, the default is the project's head commit (or nil). In such case, we # don't need to call `YamlProcessor.verify_project_sha!`, which prevents redundant calls to Gitaly. - @verify_project_sha = sha.present? + @verify_project_sha = verify_project_sha && sha.present? @sha = sha || project&.repository&.commit&.sha end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index ee1da82f285..9032faa66d4 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -140,7 +140,10 @@ module Gitlab signatures: signatures, project_id: @project.id, found_by_pipeline: report.pipeline, - vulnerability_finding_signatures_enabled: @signatures_enabled)) + vulnerability_finding_signatures_enabled: @signatures_enabled, + cvss: data['cvss'] || [] + ) + ) end def create_signatures(tracking) diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index d95ecff85cd..5b8abccc6d4 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -6,7 +6,7 @@ module Gitlab module Test class Junit JunitParserError = Class.new(Gitlab::Ci::Parsers::ParserError) - ATTACHMENT_TAG_REGEX = /\[\[ATTACHMENT\|(?<path>.+?)\]\]/.freeze + ATTACHMENT_TAG_REGEX = /\[\[ATTACHMENT\|(?<path>.+?)\]\]/ def parse!(xml_data, test_report, job:) test_suite = test_report.get_suite(job.test_suite_name) @@ -64,13 +64,16 @@ module Gitlab end def create_test_case(data, test_suite, job) + system_out = data.key?('system_out') ? "System Out:\n\n#{data['system_out']}" : nil + system_err = data.key?('system_err') ? "System Err:\n\n#{data['system_err']}" : nil + if data.key?('failure') status = ::Gitlab::Ci::Reports::TestCase::STATUS_FAILED - system_output = data['failure'] || data['system_err'] + system_output = [data['failure'], system_out, system_err].compact.join("\n\n") attachment = attachment_path(data['system_out']) elsif data.key?('error') status = ::Gitlab::Ci::Reports::TestCase::STATUS_ERROR - system_output = data['error'] || data['system_err'] + system_output = [data['error'], system_out, system_err].compact.join("\n\n") attachment = attachment_path(data['system_out']) elsif data.key?('skipped') status = ::Gitlab::Ci::Reports::TestCase::STATUS_SKIPPED diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index 76dfb4cbd87..152ea700eb7 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -7,7 +7,7 @@ module Gitlab class Skip < Chain::Base include ::Gitlab::Utils::StrongMemoize - SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i.freeze + SKIP_PATTERN = /\[(ci[ _-]skip|skip[ _-]ci)\]/i def perform! if skipped? diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 1939b1ff395..c89f9933616 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -19,7 +19,7 @@ module Gitlab end if project.import_in_progress? - return error('Import in progress') + return error('You cannot run pipelines before project import is complete.') end unless allowed_to_create_pipeline? diff --git a/lib/gitlab/ci/pipeline/expression.rb b/lib/gitlab/ci/pipeline/expression.rb index 61d392121d8..a7b82395b6d 100644 --- a/lib/gitlab/ci/pipeline/expression.rb +++ b/lib/gitlab/ci/pipeline/expression.rb @@ -5,7 +5,6 @@ module Gitlab module Pipeline module Expression ExpressionError = Class.new(StandardError) - RuntimeError = Class.new(ExpressionError) end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb index 422735bd104..70d439e2d20 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb @@ -6,7 +6,7 @@ module Gitlab module Expression module Lexeme class And < Lexeme::LogicalOperator - PATTERN = /&&/.freeze + PATTERN = /&&/ def evaluate(variables = {}) @left.evaluate(variables) && @right.evaluate(variables) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb index d35be12c996..9a45105eeaf 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb @@ -6,7 +6,7 @@ module Gitlab module Expression module Lexeme class Equals < Lexeme::LogicalOperator - PATTERN = /==/.freeze + PATTERN = /==/ def evaluate(variables = {}) @left.evaluate(variables) == @right.evaluate(variables) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index c4f06c4686d..35e08776820 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -6,7 +6,7 @@ module Gitlab module Expression module Lexeme class Matches < Lexeme::LogicalOperator - PATTERN = /=~/.freeze + PATTERN = /=~/ def evaluate(variables = {}) text = @left.evaluate(variables) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb index 64485a7e6b3..54ae3b0c369 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb @@ -6,7 +6,7 @@ module Gitlab module Expression module Lexeme class NotEquals < Lexeme::LogicalOperator - PATTERN = /!=/.freeze + PATTERN = /!=/ def evaluate(variables = {}) @left.evaluate(variables) != @right.evaluate(variables) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb index 99d9206da74..4cd9e3f3572 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb @@ -6,7 +6,7 @@ module Gitlab module Expression module Lexeme class NotMatches < Lexeme::LogicalOperator - PATTERN = /\!~/.freeze + PATTERN = /\!~/ def evaluate(variables = {}) text = @left.evaluate(variables) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb index e7f7945532b..89b7e0b102e 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb @@ -6,7 +6,7 @@ module Gitlab module Expression module Lexeme class Null < Lexeme::Value - PATTERN = /null/.freeze + PATTERN = /null/ def initialize(value = nil) super diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb index c7d653ac859..1a7b619c49c 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb @@ -6,7 +6,7 @@ module Gitlab module Expression module Lexeme class Or < Lexeme::LogicalOperator - PATTERN = /\|\|/.freeze + PATTERN = /\|\|/ def evaluate(variables = {}) @left.evaluate(variables) || @right.evaluate(variables) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb index b0ca26c9f5d..29b5e47a65f 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb @@ -6,7 +6,7 @@ module Gitlab module Expression module Lexeme class ParenthesisClose < Lexeme::Operator - PATTERN = /\)/.freeze + PATTERN = /\)/ def self.type :parenthesis_close diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb index 924fe0663ab..80f92609154 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb @@ -6,7 +6,7 @@ module Gitlab module Expression module Lexeme class ParenthesisOpen < Lexeme::Operator - PATTERN = /\(/.freeze + PATTERN = /\(/ def self.type :parenthesis_open diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index cd4106b16bb..17fe82b2236 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -5,23 +5,17 @@ module Gitlab module Pipeline module Expression module Lexeme - require_dependency 're2' - class Pattern < Lexeme::Value - PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze + PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*} def initialize(regexp) super(regexp.gsub(%r{\\/}, '/')) - unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value) - raise Lexer::SyntaxError, 'Invalid regular expression!' - end + raise Lexer::SyntaxError, 'Invalid regular expression!' unless cached_regexp.valid? end def evaluate(variables = {}) - Gitlab::UntrustedRegexp::RubySyntax.fabricate!(@value) - rescue RegexpError - raise Expression::RuntimeError, 'Invalid regular expression!' + cached_regexp.expression end def inspect @@ -47,6 +41,12 @@ module Gitlab new_pattern.evaluate(variables) end + + private + + def cached_regexp + @cached_regexp ||= RegularExpression.new(@value) + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern/regular_expression.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern/regular_expression.rb new file mode 100644 index 00000000000..5b771abf4ba --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern/regular_expression.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class Pattern + require_dependency 're2' + class RegularExpression + include Gitlab::Utils::StrongMemoize + + attr_reader :value + + def initialize(value) + @value = value + end + + def expression + Gitlab::SafeRequestStore.fetch("#{self.class}#unsafe_regexp:#{value}") do + Gitlab::UntrustedRegexp::RubySyntax.fabricate!(value) + end + end + strong_memoize_attr :expression + + def valid? + !!expression + rescue RegexpError + false + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index 798cea34db6..c43150125b7 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -6,7 +6,7 @@ module Gitlab module Expression module Lexeme class String < Lexeme::Value - PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze + PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/ def evaluate(variables = {}) @value.to_s diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index 6da88fd287e..2ecd50d32e4 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -6,7 +6,7 @@ module Gitlab module Expression module Lexeme class Variable < Lexeme::Value - PATTERN = /\$(?<name>\w+)/.freeze + PATTERN = /\$(?<name>\w+)/ def evaluate(variables = {}) unless variables.is_a?(ActiveSupport::HashWithIndifferentAccess) diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index d439149158a..fa8494483d3 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -30,12 +30,13 @@ module Gitlab attr_reader :project_id attr_reader :original_data attr_reader :found_by_pipeline + attr_reader :cvss delegate :file_path, :start_line, :end_line, to: :location alias_method :cve, :compare_key - def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false, found_by_pipeline: nil) # rubocop:disable Metrics/ParameterLists + def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false, found_by_pipeline: nil, cvss: []) # rubocop:disable Metrics/ParameterLists @compare_key = compare_key @confidence = confidence @identifiers = identifiers @@ -57,6 +58,7 @@ module Gitlab @project_id = project_id @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled @found_by_pipeline = found_by_pipeline + @cvss = cvss @project_fingerprint = generate_project_fingerprint end diff --git a/lib/gitlab/ci/status/canceled.rb b/lib/gitlab/ci/status/canceled.rb index f173964b36c..a3376692570 100644 --- a/lib/gitlab/ci/status/canceled.rb +++ b/lib/gitlab/ci/status/canceled.rb @@ -5,7 +5,7 @@ module Gitlab module Status class Canceled < Status::Core def text - s_('CiStatusText|canceled') + s_('CiStatusText|Canceled') end def label diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index f60f5243666..c5306de830b 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -38,6 +38,10 @@ module Gitlab raise NotImplementedError end + def name + self.class.name.demodulize.underscore.upcase + end + def group self.class.name.demodulize.underscore end diff --git a/lib/gitlab/ci/status/created.rb b/lib/gitlab/ci/status/created.rb index 33e67314d93..9ad4b2f079e 100644 --- a/lib/gitlab/ci/status/created.rb +++ b/lib/gitlab/ci/status/created.rb @@ -5,7 +5,7 @@ module Gitlab module Status class Created < Status::Core def text - s_('CiStatusText|created') + s_('CiStatusText|Created') end def label diff --git a/lib/gitlab/ci/status/failed.rb b/lib/gitlab/ci/status/failed.rb index 215d27734a7..cb498f72ffe 100644 --- a/lib/gitlab/ci/status/failed.rb +++ b/lib/gitlab/ci/status/failed.rb @@ -5,7 +5,7 @@ module Gitlab module Status class Failed < Status::Core def text - s_('CiStatusText|failed') + s_('CiStatusText|Failed') end def label diff --git a/lib/gitlab/ci/status/manual.rb b/lib/gitlab/ci/status/manual.rb index eb376df5f22..02e65dd1f4c 100644 --- a/lib/gitlab/ci/status/manual.rb +++ b/lib/gitlab/ci/status/manual.rb @@ -5,7 +5,7 @@ module Gitlab module Status class Manual < Status::Core def text - s_('CiStatusText|manual') + s_('CiStatusText|Manual') end def label diff --git a/lib/gitlab/ci/status/pending.rb b/lib/gitlab/ci/status/pending.rb index 4280ad84534..ddbdf94c089 100644 --- a/lib/gitlab/ci/status/pending.rb +++ b/lib/gitlab/ci/status/pending.rb @@ -5,7 +5,7 @@ module Gitlab module Status class Pending < Status::Core def text - s_('CiStatusText|pending') + s_('CiStatusText|Pending') end def label diff --git a/lib/gitlab/ci/status/pipeline/blocked.rb b/lib/gitlab/ci/status/pipeline/blocked.rb index ed13a439be0..2e01f4948a9 100644 --- a/lib/gitlab/ci/status/pipeline/blocked.rb +++ b/lib/gitlab/ci/status/pipeline/blocked.rb @@ -6,7 +6,7 @@ module Gitlab module Pipeline class Blocked < Status::Extended def text - s_('CiStatusText|blocked') + s_('CiStatusText|Blocked') end def label diff --git a/lib/gitlab/ci/status/pipeline/delayed.rb b/lib/gitlab/ci/status/pipeline/delayed.rb index e61acdcd167..47048afbe1d 100644 --- a/lib/gitlab/ci/status/pipeline/delayed.rb +++ b/lib/gitlab/ci/status/pipeline/delayed.rb @@ -6,7 +6,7 @@ module Gitlab module Pipeline class Delayed < Status::Extended def text - s_('CiStatusText|delayed') + s_('CiStatusText|Delayed') end def label diff --git a/lib/gitlab/ci/status/preparing.rb b/lib/gitlab/ci/status/preparing.rb index e59d1d2eed1..e29b5416e8d 100644 --- a/lib/gitlab/ci/status/preparing.rb +++ b/lib/gitlab/ci/status/preparing.rb @@ -5,7 +5,7 @@ module Gitlab module Status class Preparing < Status::Core def text - s_('CiStatusText|preparing') + s_('CiStatusText|Preparing') end def label diff --git a/lib/gitlab/ci/status/running.rb b/lib/gitlab/ci/status/running.rb index eed1983e60e..dc36e62e2a3 100644 --- a/lib/gitlab/ci/status/running.rb +++ b/lib/gitlab/ci/status/running.rb @@ -5,11 +5,11 @@ module Gitlab module Status class Running < Status::Core def text - s_('CiStatus|running') + s_('CiStatusText|Running') end def label - s_('CiStatus|running') + s_('CiStatusLabel|running') end def icon diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb index 8526becfef9..a3797c5c8d7 100644 --- a/lib/gitlab/ci/status/scheduled.rb +++ b/lib/gitlab/ci/status/scheduled.rb @@ -5,7 +5,7 @@ module Gitlab module Status class Scheduled < Status::Core def text - s_('CiStatusText|scheduled') + s_('CiStatusText|Scheduled') end def label diff --git a/lib/gitlab/ci/status/skipped.rb b/lib/gitlab/ci/status/skipped.rb index 238aa3ab4f9..4263536552b 100644 --- a/lib/gitlab/ci/status/skipped.rb +++ b/lib/gitlab/ci/status/skipped.rb @@ -5,7 +5,7 @@ module Gitlab module Status class Skipped < Status::Core def text - s_('CiStatusText|skipped') + s_('CiStatusText|Skipped') end def label diff --git a/lib/gitlab/ci/status/success.rb b/lib/gitlab/ci/status/success.rb index 2a10e60414e..9389138e034 100644 --- a/lib/gitlab/ci/status/success.rb +++ b/lib/gitlab/ci/status/success.rb @@ -5,7 +5,7 @@ module Gitlab module Status class Success < Status::Core def text - s_('CiStatusText|passed') + s_('CiStatusText|Passed') end def label diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index 84a0e52f518..91f0ba1a58f 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -9,7 +9,7 @@ module Gitlab # class SuccessWarning < Status::Extended def text - s_('CiStatusText|warning') + s_('CiStatusText|Warning') end def label @@ -20,6 +20,10 @@ module Gitlab 'status_warning' end + def name + 'SUCCESS_WITH_WARNINGS' + end + def group 'success-with-warnings' end diff --git a/lib/gitlab/ci/status/waiting_for_resource.rb b/lib/gitlab/ci/status/waiting_for_resource.rb index 9ced0aadb88..5714a68cac8 100644 --- a/lib/gitlab/ci/status/waiting_for_resource.rb +++ b/lib/gitlab/ci/status/waiting_for_resource.rb @@ -5,7 +5,7 @@ module Gitlab module Status class WaitingForResource < Status::Core def text - s_('CiStatusText|waiting') + s_('CiStatusText|Waiting') end def label @@ -20,6 +20,10 @@ module Gitlab 'favicon_status_pending' end + def name + 'WAITING_FOR_RESOURCE' + end + def group 'waiting-for-resource' end diff --git a/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml index b4ccf96b859..3132535ef6b 100644 --- a/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Code-Quality.gitlab-ci.yml @@ -1,2 +1,2 @@ include: - template: Jobs/Code-Quality.gitlab-ci.yml + - template: Jobs/Code-Quality.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml b/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml index 48c9422b469..356062c734e 100644 --- a/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml @@ -8,7 +8,7 @@ # See https://docs.gitlab.com/ee/ci/yaml/signing_examples.html for more details. include: - template: Docker.gitlab-ci.yml + - template: Docker.gitlab-ci.yml docker-build: variables: diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml index 1aa346aec67..416f424dfa5 100644 --- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -11,7 +11,7 @@ docker-build: # Use the official docker image. - image: docker:latest + image: docker:cli stage: build services: - docker:dind diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 07bc3fbe795..2d04c97b32e 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.41.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.44.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 07bc3fbe795..2d04c97b32e 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.41.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.44.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index f9440bfe904..45547b87eb6 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,9 @@ code_quality: command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] variables: DOCKER_DRIVER: overlay2 + DOCKER_CERT_PATH: "" DOCKER_TLS_CERTDIR: "" + DOCKER_TLS_VERIFY: "" CODE_QUALITY_IMAGE_TAG: "0.96.0" CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG" DOCKER_SOCKET_PATH: /var/run/docker.sock diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index e9ba938142d..4d53b92763a 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.56.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index eaaf171e4b5..390824e8e49 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.56.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index d2e448fb6a1..a9681c0f927 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.56.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index d53f3ddcad4..c19a08bd11d 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -12,15 +12,10 @@ image: python:latest variables: PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip" -# Pip's cache doesn't store the python packages # https://pip.pypa.io/en/stable/topics/caching/ -# -# If you want to also cache the installed packages, you have to install -# them in a virtualenv and cache it as well. cache: paths: - .cache/pip - - venv/ before_script: - python --version ; pip --version # For debugging diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 879d6a7a468..d6384f59bc1 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -2,4 +2,4 @@ # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/381665 include: - template: Jobs/Container-Scanning.gitlab-ci.yml + - template: Jobs/Container-Scanning.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml index 7a4f451314e..f4fd9e97665 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml @@ -2,4 +2,4 @@ # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/381665 include: - template: Jobs/Container-Scanning.latest.gitlab-ci.yml + - template: Jobs/Container-Scanning.latest.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 1785d4216e7..2055b5e181f 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -2,4 +2,4 @@ # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977 include: - template: Jobs/Dependency-Scanning.gitlab-ci.yml + - template: Jobs/Dependency-Scanning.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index a99fe4a6dcf..0fe544b2c84 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -2,4 +2,4 @@ # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977 include: - template: Jobs/License-Scanning.gitlab-ci.yml + - template: Jobs/License-Scanning.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml index 2207d4ec17a..4cc51c01b63 100644 --- a/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST-IaC.gitlab-ci.yml @@ -1,2 +1,2 @@ include: - template: Jobs/SAST-IaC.gitlab-ci.yml + - template: Jobs/SAST-IaC.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml index 8c0d72ff282..a411fc03122 100644 --- a/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST-IaC.latest.gitlab-ci.yml @@ -1,2 +1,2 @@ include: - template: Jobs/SAST-IaC.latest.gitlab-ci.yml + - template: Jobs/SAST-IaC.latest.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 77ce813dd4f..6c25d628d55 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -2,4 +2,4 @@ # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977 include: - template: Jobs/SAST.gitlab-ci.yml + - template: Jobs/SAST.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index d4ea7165d0a..353d523daf3 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -2,4 +2,4 @@ # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/292977 include: - template: Jobs/Secret-Detection.gitlab-ci.yml + - template: Jobs/Secret-Detection.gitlab-ci.yml diff --git a/lib/gitlab/ci/trace/section_parser.rb b/lib/gitlab/ci/trace/section_parser.rb index f33f8cc56c1..a6c1bf28f24 100644 --- a/lib/gitlab/ci/trace/section_parser.rb +++ b/lib/gitlab/ci/trace/section_parser.rb @@ -74,7 +74,7 @@ module Gitlab end def beginning_of_section_regex - @beginning_of_section_regex ||= /section_/.freeze + @beginning_of_section_regex ||= /section_/ end def find_next_marker(scanner) diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 73452d83bce..2334db0718f 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -7,7 +7,7 @@ module Gitlab class Item include Gitlab::Utils::StrongMemoize - VARIABLES_REGEXP = /\$\$|%%|\$(?<key>[a-zA-Z_][a-zA-Z0-9_]*)|\${\g<key>?}|%\g<key>%/.freeze.freeze + VARIABLES_REGEXP = /\$\$|%%|\$(?<key>[a-zA-Z_][a-zA-Z0-9_]*)|\${\g<key>?}|%\g<key>%/ VARIABLE_REF_CHARS = %w[$ %].freeze def initialize(key:, value:, public: true, file: false, masked: false, raw: false) @@ -34,6 +34,10 @@ module Gitlab @variable.fetch(:file) end + def masked? + @variable.fetch(:masked) + end + def [](key) @variable.fetch(key) end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 289f41b4ec7..cf5755242e2 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -209,7 +209,8 @@ module Gitlab return unless project && sha && project.repository_exists? && project.commit(sha) unless project_ref_contains_sha? - error!('Could not validate configuration. Config originates from external project') + error!('Could not validate configuration. The configuration originates from an external ' \ + 'project or a commit not associated with a Git reference (a detached commit)') end end diff --git a/lib/gitlab/cleanup/project_uploads.rb b/lib/gitlab/cleanup/project_uploads.rb index 7f24b2f78b0..6feaab2a791 100644 --- a/lib/gitlab/cleanup/project_uploads.rb +++ b/lib/gitlab/cleanup/project_uploads.rb @@ -93,7 +93,7 @@ module Gitlab end class ProjectUploadPath - PROJECT_FULL_PATH_REGEX = %r{\A#{FileUploader.root}/(.+)/(\h+/[^/]+)\z}.freeze + PROJECT_FULL_PATH_REGEX = %r{\A#{FileUploader.root}/(.+)/(\h+/[^/]+)\z} attr_reader :full_path, :upload_path diff --git a/lib/gitlab/color.rb b/lib/gitlab/color.rb index c31c8cb5876..4be78f59bd3 100644 --- a/lib/gitlab/color.rb +++ b/lib/gitlab/color.rb @@ -2,7 +2,7 @@ module Gitlab class Color - PATTERN = /\A\#(?:[0-9A-Fa-f]{3}){1,2}\Z/.freeze + PATTERN = /\A\#(?:[0-9A-Fa-f]{3}){1,2}\Z/ def initialize(value) @value = value&.strip&.freeze diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb index ec67d65c526..1f70afbfb75 100644 --- a/lib/gitlab/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb @@ -12,7 +12,7 @@ module Gitlab if parser && parser.respond_to?(:validate_duration) parser.validate_duration(value) else - ChronicDuration.parse(value, use_complete_matcher: true) + ChronicDuration.parse(value) end rescue ChronicDuration::DurationParseError false @@ -24,12 +24,7 @@ module Gitlab if parser && parser.respond_to?(:validate_duration_limit) parser.validate_duration_limit(value, limit) else - ChronicDuration.parse( - value, use_complete_matcher: true - ).second.from_now < - ChronicDuration.parse( - limit, use_complete_matcher: true - ).second.from_now + ChronicDuration.parse(value).second.from_now < ChronicDuration.parse(limit).second.from_now end rescue ChronicDuration::DurationParseError false diff --git a/lib/gitlab/config/loader/multi_doc_yaml.rb b/lib/gitlab/config/loader/multi_doc_yaml.rb index 084d32a85bc..5db1cc9a5d5 100644 --- a/lib/gitlab/config/loader/multi_doc_yaml.rb +++ b/lib/gitlab/config/loader/multi_doc_yaml.rb @@ -6,7 +6,7 @@ module Gitlab class MultiDocYaml include Gitlab::Utils::StrongMemoize - MULTI_DOC_DIVIDER = /^---\s+/.freeze + MULTI_DOC_DIVIDER = /^---\s+/ def initialize(config, max_documents:, additional_permitted_classes: [], reject_empty: false) @config = config diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 87a6d4ada70..f7c9d95c53c 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -86,7 +86,7 @@ module Gitlab def add_browsersdk_tracking(directives) return if directives.blank? - return unless Gitlab.com? && Feature.enabled?(:browsersdk_tracking) && ENV['GITLAB_ANALYTICS_URL'].present? + return unless Gitlab.com? && ENV['GITLAB_ANALYTICS_URL'].present? default_connect_src = directives['connect-src'] || directives['default-src'] connect_src_values = Array.wrap(default_connect_src) | [ENV['GITLAB_ANALYTICS_URL']] diff --git a/lib/gitlab/database/background_migration/batch_optimizer.rb b/lib/gitlab/database/background_migration/batch_optimizer.rb index 9eb456f6e2e..e99b63c0e4b 100644 --- a/lib/gitlab/database/background_migration/batch_optimizer.rb +++ b/lib/gitlab/database/background_migration/batch_optimizer.rb @@ -17,7 +17,7 @@ module Gitlab class BatchOptimizer # Target time efficiency for a job # Time efficiency is defined as: job duration / interval - TARGET_EFFICIENCY = (0.9..0.95).freeze + TARGET_EFFICIENCY = (0.9..0.95) # Lower and upper bound for the batch size MIN_BATCH_SIZE = 1_000 diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 0bd357b7730..31ceb898eee 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -87,24 +87,37 @@ module Gitlab # rubocop:enable Gitlab/DocUrl end - private_class_method def self.cross_access_allowed?(type, table_schemas) + def self.cross_joins_allowed?(table_schemas, all_tables) + return true unless table_schemas.many? + table_schemas.any? do |schema| - extra_schemas = table_schemas - [schema] - extra_schemas -= Gitlab::Database.all_gitlab_schemas[schema]&.public_send(type) || [] # rubocop:disable GitlabSecurity/PublicSend - extra_schemas.empty? + schema_info = Gitlab::Database.all_gitlab_schemas[schema] + next false unless schema_info + + schema_info.allow_cross_joins?(table_schemas, all_tables) end end - def self.cross_joins_allowed?(table_schemas) - table_schemas.empty? || self.cross_access_allowed?(:allow_cross_joins, table_schemas) - end + def self.cross_transactions_allowed?(table_schemas, all_tables) + return true unless table_schemas.many? + + table_schemas.any? do |schema| + schema_info = Gitlab::Database.all_gitlab_schemas[schema] + next false unless schema_info - def self.cross_transactions_allowed?(table_schemas) - table_schemas.empty? || self.cross_access_allowed?(:allow_cross_transactions, table_schemas) + schema_info.allow_cross_transactions?(table_schemas, all_tables) + end end - def self.cross_foreign_key_allowed?(table_schemas) - self.cross_access_allowed?(:allow_cross_foreign_keys, table_schemas) + def self.cross_foreign_key_allowed?(table_schemas, all_tables) + return true if table_schemas.one? + + table_schemas.any? do |schema| + schema_info = Gitlab::Database.all_gitlab_schemas[schema] + next false unless schema_info + + schema_info.allow_cross_foreign_keys?(table_schemas, all_tables) + end end def self.dictionary_paths diff --git a/lib/gitlab/database/gitlab_schema_info.rb b/lib/gitlab/database/gitlab_schema_info.rb index 34b89cb9006..20d2b31a65c 100644 --- a/lib/gitlab/database/gitlab_schema_info.rb +++ b/lib/gitlab/database/gitlab_schema_info.rb @@ -2,6 +2,11 @@ module Gitlab module Database + GitlabSchemaInfoAllowCross = Struct.new( + :specific_tables, + keyword_init: true + ) + GitlabSchemaInfo = Struct.new( :name, :description, @@ -14,15 +19,76 @@ module Gitlab def initialize(*) super self.name = name.to_sym - self.allow_cross_joins = allow_cross_joins&.map(&:to_sym)&.freeze - self.allow_cross_transactions = allow_cross_transactions&.map(&:to_sym)&.freeze - self.allow_cross_foreign_keys = allow_cross_foreign_keys&.map(&:to_sym)&.freeze + self.allow_cross_joins = convert_array_to_hash(allow_cross_joins) + self.allow_cross_transactions = convert_array_to_hash(allow_cross_transactions) + self.allow_cross_foreign_keys = convert_array_to_hash(allow_cross_foreign_keys) end def self.load_file(yaml_file) content = YAML.load_file(yaml_file) new(**content.deep_symbolize_keys.merge(file_path: yaml_file)) end + + def allow_cross_joins?(table_schemas, all_tables) + allowed_schemas = allow_cross_joins || {} + + allowed_for?(allowed_schemas, table_schemas, all_tables) + end + + def allow_cross_transactions?(table_schemas, all_tables) + allowed_schemas = allow_cross_transactions || {} + + allowed_for?(allowed_schemas, table_schemas, all_tables) + end + + def allow_cross_foreign_keys?(table_schemas, all_tables) + allowed_schemas = allow_cross_foreign_keys || {} + + allowed_for?(allowed_schemas, table_schemas, all_tables) + end + + private + + def allowed_for?(allowed_schemas, table_schemas, all_tables) + denied_schemas = table_schemas - [name] + denied_schemas -= allowed_schemas.keys + return false unless denied_schemas.empty? + + all_tables.all? do |table| + table_schema = ::Gitlab::Database::GitlabSchema.table_schema!(table) + allowed_tables = allowed_schemas[table_schema] + + allowed_tables.nil? || allowed_tables.specific_tables.include?(table) + end + end + + # Convert from: + # - schema_a + # - schema_b: + # specific_tables: + # - table_b_of_schema_b + # - table_c_of_schema_b + # + # To: + # { :schema_a => nil, + # :schema_b => { specific_tables : [:table_b_of_schema_b, :table_c_of_schema_b] } + # } + # + def convert_array_to_hash(subject) + result = {} + + subject&.each do |item| + if item.is_a?(Hash) + item.each do |key, value| + result[key.to_sym] = GitlabSchemaInfoAllowCross.new(value || {}) + end + else + result[item.to_sym] = nil + end + end + + result.freeze + end end end end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index 1f9ab1cfe98..3d11f0f88c1 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -24,7 +24,7 @@ module Gitlab MAX_DISCOVERY_RETRIES = 3 DISCOVERY_THREAD_REFRESH_DELTA = 5 - RETRY_DELAY_RANGE = (0.1..0.2).freeze + RETRY_DELAY_RANGE = (0.1..0.2) RECORD_TYPES = { 'A' => Net::DNS::A, @@ -151,13 +151,7 @@ module Gitlab # started just before we added the new hosts it will use an old # host/connection. While this connection will be checked in and out, # it won't be explicitly disconnected. - if Gitlab::Utils.to_boolean(ENV['LOAD_BALANCER_PARALLEL_DISCONNECT'], default: false) - disconnect_old_hosts(old_hosts) - else - old_hosts.each do |host| - host.disconnect!(timeout: disconnect_timeout) - end - end + disconnect_old_hosts(old_hosts) end # Returns an Array containing: diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb index fbb71c1ccfd..41044816de9 100644 --- a/lib/gitlab/database/migration.rb +++ b/lib/gitlab/database/migration.rb @@ -34,7 +34,7 @@ module Gitlab # to indicate backwards-compatible or otherwise minor changes (e.g. a Rails version bump). # However, this hasn't been strictly formalized yet. - class V1_0 < ActiveRecord::Migration[6.1] # rubocop:disable Naming/ClassAndModuleCamelCase + class V1_0 < ActiveRecord::Migration[6.1] include LockRetriesConcern include Gitlab::Database::MigrationHelpers::V2 include Gitlab::Database::MigrationHelpers::AnnounceDatabase @@ -47,11 +47,11 @@ module Gitlab end end - class V2_0 < V1_0 # rubocop:disable Naming/ClassAndModuleCamelCase + class V2_0 < V1_0 include Gitlab::Database::MigrationHelpers::RestrictGitlabSchema end - class V2_1 < V2_0 # rubocop:disable Naming/ClassAndModuleCamelCase + class V2_1 < V2_0 include Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables include Gitlab::Database::Migrations::RunnerBackoff::MigrationHelpers end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 60cec12b4b5..efcceafda90 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -567,8 +567,8 @@ module Gitlab # table - The table containing the column. # column - The name of the column to change. # new_type - The new column type. - def cleanup_concurrent_column_type_change(table, column) - temp_column = "#{column}_for_type_change" + def cleanup_concurrent_column_type_change(table, column, temp_column: nil) + temp_column ||= "#{column}_for_type_change" transaction do # This has to be performed in a transaction as otherwise we might have @@ -586,10 +586,10 @@ module Gitlab # type_cast_function - Required if the conversion back to the original type is not automatic # batch_column_name - option for tables without a primary key, in this case # another unique integer column can be used. Example: :user_id - def undo_cleanup_concurrent_column_type_change(table, column, old_type, type_cast_function: nil, batch_column_name: :id, limit: nil) + def undo_cleanup_concurrent_column_type_change(table, column, old_type, type_cast_function: nil, batch_column_name: :id, limit: nil, temp_column: nil) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! - temp_column = "#{column}_for_type_change" + temp_column ||= "#{column}_for_type_change" # Using a descriptive name that includes orinal column's name risks # taking us above the 63 character limit, so we use a hash @@ -751,7 +751,7 @@ module Gitlab trigger_name = rename_trigger_name(table, columns, temporary_columns) remove_rename_triggers(table, trigger_name) - temporary_columns.each { |column| remove_column(table, column) } + temporary_columns.each { |column| remove_column(table, column, if_exists: true) } end alias_method :cleanup_conversion_of_integer_to_bigint, :revert_initialize_conversion_of_integer_to_bigint @@ -909,6 +909,11 @@ module Gitlab name = index.name.gsub(old, new) + if name.length > 63 + digest = Digest::SHA256.hexdigest(name).first(10) + name = "idx_copy_#{digest}" + end + options = { unique: index.unique, name: name, @@ -1204,6 +1209,10 @@ into similar problems in the future (e.g. when new tables are created). end end + def lock_tables(*tables, mode: :access_exclusive) + execute("LOCK TABLE #{tables.join(', ')} IN #{mode.to_s.upcase.tr('_', ' ')} MODE") + end + private def multiple_columns(columns, separator: ', ') diff --git a/lib/gitlab/database/migration_helpers/swapping.rb b/lib/gitlab/database/migration_helpers/swapping.rb new file mode 100644 index 00000000000..6d19f8002d8 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/swapping.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module Swapping + def reset_trigger_function(function_name) + execute("ALTER FUNCTION #{quote_table_name(function_name)} RESET ALL") + end + + def swap_columns(table, column1, column2) + ::Gitlab::Database::Migrations::SwapColumns.new( + migration_context: self, + table: table, + column1: column1, + column2: column2 + ).execute + end + + def swap_columns_default(table, column1, column2) + ::Gitlab::Database::Migrations::SwapColumnsDefault.new( + migration_context: self, + table: table, + column1: column1, + column2: column2 + ).execute + end + + def swap_foreign_keys(table, foreign_key1, foreign_key2) + rename_constraint(table, foreign_key1, :temp_name_for_renaming) + rename_constraint(table, foreign_key2, foreign_key1) + rename_constraint(table, :temp_name_for_renaming, foreign_key2) + end + + def swap_indexes(table, index1, index2) + identifier = "index_#{index1}_on_#{table}" + # Check Gitlab::Database::MigrationHelpers#concurrent_foreign_key_name() + # for info on why we use a hash + hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) + + temp_index = "temp_#{hashed_identifier}" + + rename_index(table, index1, temp_index) + rename_index(table, index2, index1) + rename_index(table, temp_index, index2) + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb index 07e22963177..7cfafa1a6a6 100644 --- a/lib/gitlab/database/migration_helpers/v2.rb +++ b/lib/gitlab/database/migration_helpers/v2.rb @@ -43,7 +43,7 @@ module Gitlab end end - t.instance_eval(&block) unless block.nil? + yield t unless block.nil? end end diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index efb1957d5e7..64cde273a59 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -38,6 +38,10 @@ module Gitlab # batch_class_name - The name of the class that will be called to find the range of each next batch # batch_size - The maximum number of rows per job # sub_batch_size - The maximum number of rows processed per "iteration" within the job + # queued_migration_version - Version of the migration that queues the BBM, this is used to establish dependecies + # + # queued_migration_version is made optional temporarily to allow prior migrations to not fail, + # https://gitlab.com/gitlab-org/gitlab/-/issues/426417 will make it mandatory. # # *Returns the created BatchedMigration record* # @@ -63,6 +67,7 @@ module Gitlab batch_column_name, *job_arguments, job_interval:, + queued_migration_version: nil, batch_min_value: BATCH_MIN_VALUE, batch_max_value: nil, batch_class_name: BATCH_CLASS_NAME, @@ -113,27 +118,13 @@ module Gitlab "(given #{job_arguments.count}, expected #{migration.job_class.job_arguments_count})" end - # Below `BatchedMigration` attributes were introduced after the - # initial `batched_background_migrations` table was created, so any - # migrations that ran relying on initial table schema would not know - # about columns introduced later on because this model is not - # isolated in migrations, which is why we need to check for existence - # of these columns first. - if migration.respond_to?(:max_batch_size) - migration.max_batch_size = max_batch_size - end - - if migration.respond_to?(:total_tuple_count) - # We keep track of the estimated number of tuples to reason later - # about the overall progress of a migration. - migration.total_tuple_count = Gitlab::Database::SharedModel.using_connection(connection) do - Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate - end - end - - if migration.respond_to?(:gitlab_schema) - migration.gitlab_schema = gitlab_schema - end + assign_attribtues_safely( + migration, + max_batch_size, + batch_table_name, + gitlab_schema, + queued_migration_version + ) migration.save! migration @@ -244,6 +235,33 @@ module Gitlab "\n\n" \ "\thttps://docs.gitlab.com/ee/update/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished" end + + private + + # Below `BatchedMigration` attributes were introduced after the + # initial `batched_background_migrations` table was created, so any + # migrations that ran relying on initial table schema would not know + # about columns introduced later on because this model is not + # isolated in migrations, which is why we need to check for existence + # of these columns first. + def assign_attribtues_safely(migration, max_batch_size, batch_table_name, gitlab_schema, queued_migration_version) + # We keep track of the estimated number of tuples in 'total_tuple_count' to reason later + # about the overall progress of a migration. + safe_attributes_value = { + max_batch_size: max_batch_size, + total_tuple_count: Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate + end, + gitlab_schema: gitlab_schema, + queued_migration_version: queued_migration_version + } + + # rubocop:disable GitlabSecurity/PublicSend + safe_attributes_value.each do |safe_attribute, value| + migration.public_send("#{safe_attribute}=", value) if migration.respond_to?(safe_attribute) + end + # rubocop:enable GitlabSecurity/PublicSend + end end end end diff --git a/lib/gitlab/database/migrations/milestone_mixin.rb b/lib/gitlab/database/migrations/milestone_mixin.rb new file mode 100644 index 00000000000..10bc0c192e7 --- /dev/null +++ b/lib/gitlab/database/migrations/milestone_mixin.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module MilestoneMixin + extend ActiveSupport::Concern + include Gitlab::ClassAttributes + + MilestoneNotSetError = Class.new(StandardError) + + class_methods do + def milestone(milestone_str = nil) + if milestone_str.present? + set_class_attribute(:migration_milestone, milestone_str) + else + get_class_attribute(:migration_milestone) + end + end + end + + def initialize(name = class_name, version = nil, type = nil) + raise MilestoneNotSetError, "Milestone is not set for #{self.class.name}" if milestone.nil? + + super(name, version) + @version = Gitlab::Database::Migrations::Version.new(version, milestone, type) + end + + def milestone # rubocop:disable Lint/DuplicateMethods + @milestone ||= self.class.milestone + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/observers/query_statistics.rb b/lib/gitlab/database/migrations/observers/query_statistics.rb index 2d026f0c8d2..b0797cd4f3f 100644 --- a/lib/gitlab/database/migrations/observers/query_statistics.rb +++ b/lib/gitlab/database/migrations/observers/query_statistics.rb @@ -20,7 +20,13 @@ module Gitlab return unless enabled? observation.query_statistics = connection.execute(<<~SQL) - SELECT query, calls, total_time, max_time, mean_time, rows + SELECT + query, + calls, + total_exec_time + total_plan_time AS total_time, + max_exec_time + max_plan_time AS max_time, + mean_exec_time + mean_plan_time AS mean_time, + "rows" FROM pg_stat_statements WHERE pg_get_userbyid(userid) = current_user ORDER BY total_time DESC diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index dc9ea304aac..f640d6fcf75 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -7,7 +7,7 @@ module Gitlab BASE_RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze METADATA_FILENAME = 'metadata.json' SCHEMA_VERSION = 4 # Version of the output format produced by the runner - POST_MIGRATION_MATCHER = %r{db/post_migrate/}.freeze + POST_MIGRATION_MATCHER = %r{db/post_migrate/} class << self def up(database:, legacy_mode: false) diff --git a/lib/gitlab/database/migrations/swap_columns.rb b/lib/gitlab/database/migrations/swap_columns.rb new file mode 100644 index 00000000000..6d0347553b3 --- /dev/null +++ b/lib/gitlab/database/migrations/swap_columns.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class SwapColumns + delegate :quote_table_name, :quote_column_name, :clear_cache!, to: :@migration_context + + def initialize(migration_context:, table:, column1:, column2:) + @migration_context = migration_context + @table = table + @column_name1 = column1 + @column_name2 = column2 + end + + def execute + rename_column(@table, @column_name1, :temp_name_for_renaming) + rename_column(@table, @column_name2, @column_name1) + rename_column(@table, :temp_name_for_renaming, @column_name2) + end + + private + + # Rails' `rename_column` will rename related indexes + # using a format e.g. `index_{TABLE_NAME}_on_{KEY1}_and_{KEY2}` + # This will break the migration if the formated index name is longer than 63 chars, e.g. + # `index_ci_pipeline_variables_on_pipeline_id_convert_to_bigint_and_key` + # Therefore, we need to duplicate what Rails has done here without the part renaming related indexes + def rename_column(table_name, column_name, column2_name) + clear_cache! + @migration_context.execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} + RENAME COLUMN #{quote_column_name(column_name)} TO #{quote_column_name(column2_name)} + SQL + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/swap_columns_default.rb b/lib/gitlab/database/migrations/swap_columns_default.rb new file mode 100644 index 00000000000..0005c606b87 --- /dev/null +++ b/lib/gitlab/database/migrations/swap_columns_default.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class SwapColumnsDefault + delegate( + :change_column_default, :quote_table_name, :quote_column_name, :column_for, + to: :@migration_context + ) + + def initialize(migration_context:, table:, column1:, column2:) + @migration_context = migration_context + @table = table + @column_name1 = column1 + @column_name2 = column2 + end + + def execute + default1 = find_default_by(@column_name1) + default2 = find_default_by(@column_name2) + return if default1 == default2 + + change_sequence_owner_if(default1[:sequence_name], @column_name2) + change_sequence_owner_if(default2[:sequence_name], @column_name1) + + change_column_default(@table, @column_name1, default2[:default]) + change_column_default(@table, @column_name2, default1[:default]) + end + + private + + def change_sequence_owner_if(sequence_name, column_name) + return if sequence_name.blank? + + @migration_context.execute(<<~SQL.squish) + ALTER SEQUENCE #{quote_table_name(sequence_name)} + OWNED BY #{quote_table_name(@table)}.#{quote_column_name(column_name)} + SQL + end + + def find_default_by(name) + column = column_for(@table, name) + if column.default_function.present? + { + default: -> { column.default_function }, + sequence_name: extract_sequence_name_from(column.default_function) + } + else + { + default: column.default + } + end + end + + def extract_sequence_name_from(expression) + expression[/nextval\('([^']+)'/, 1] + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/version.rb b/lib/gitlab/database/migrations/version.rb new file mode 100644 index 00000000000..27c4c7d0746 --- /dev/null +++ b/lib/gitlab/database/migrations/version.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class Version + InvalidTypeError = Class.new(StandardError) + + include Comparable + + TYPE_VALUES = { + regular: 0, + post: 1 + }.freeze + + attr_reader :timestamp, :milestone, :type_value + + def initialize(timestamp, milestone, type) + @timestamp = timestamp + @milestone = milestone + self.type = type + end + + def type + TYPE_VALUES.key(@type_value) + end + + def type=(value) + @type_value = TYPE_VALUES.fetch(value.to_sym) { raise InvalidTypeError } + end + + def regular? + @type_value == TYPE_VALUES[:regular] + end + + def post_deployment? + @type_value == TYPE_VALUES[:post] + end + + def <=>(other) + return 1 unless other.is_a?(self.class) + + return milestone <=> other.milestone if milestone != other.milestone + + return @type_value <=> other.type_value if @type_value != other.type_value + + @timestamp <=> other.timestamp + end + + def to_s + @timestamp.to_s + end + + def to_i + @timestamp.to_i + end + + def coerce(_other) + [-1, timestamp.to_i] + end + + def eql?(other) + (self <=> other) == 0 + end + + def ==(other) + eql?(other) + end + + def hash + [timestamp, milestone, @type_value].hash + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index cc5c49cc24a..bb70d052e3e 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -153,7 +153,6 @@ module Gitlab end def run_analyze_on_partitioned_table - return if Feature.disabled?(:database_analyze_on_partitioned_tables) return if ineligible_for_analyzing? primary_transaction(statement_timeout: STATEMENT_TIMEOUT) do diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index a9f2b963340..fb25cb70e57 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -113,7 +113,7 @@ module Gitlab schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(all_tables) schemas += ApplicationRecord.gitlab_transactions_stack - unless ::Gitlab::Database::GitlabSchema.cross_transactions_allowed?(schemas) + unless ::Gitlab::Database::GitlabSchema.cross_transactions_allowed?(schemas, all_tables) messages = [] messages << "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb deleted file mode 100644 index 2314246da55..00000000000 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -# This module can be included in migrations to make it easier to rename paths -# of `Namespace` & `Project` models certain paths would become `reserved`. -# -# If the way things are stored on the filesystem related to namespaces and -# projects ever changes. Don't update this module, or anything nested in `V1`, -# since it needs to keep functioning for all migrations using it using the state -# that the data is in at the time. Instead, create a `V2` module that implements -# the new way of reserving paths. -module Gitlab - module Database - module RenameReservedPathsMigration - module V1 - def self.included(kls) - kls.include(MigrationHelpers) - end - - def rename_wildcard_paths(one_or_more_paths) - rename_child_paths(one_or_more_paths) - paths = Array(one_or_more_paths) - RenameProjects.new(paths, self).rename_projects - end - - def rename_child_paths(one_or_more_paths) - paths = Array(one_or_more_paths) - RenameNamespaces.new(paths, self).rename_namespaces(type: :child) - end - - def rename_root_paths(paths) - paths = Array(paths) - RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level) - end - - def revert_renames - RenameProjects.new([], self).revert_renames - RenameNamespaces.new([], self).revert_renames - end - end - end - end -end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb deleted file mode 100644 index f1dc3ed74fe..00000000000 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb +++ /dev/null @@ -1,99 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module RenameReservedPathsMigration - module V1 - module MigrationClasses - module Routable - def full_path - if route && route.path.present? - @full_path ||= route.path # rubocop:disable Gitlab/ModuleWithInstanceVariables - else - update_route if persisted? - - build_full_path - end - end - - def build_full_path - if parent && path - parent.full_path + '/' + path - else - path - end - end - - def update_route - prepare_route - route.save - end - - def prepare_route - route || build_route(source: self) - route.path = build_full_path - @full_path = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables - end - end - - class Namespace < ActiveRecord::Base - include MigrationClasses::Routable - self.table_name = 'namespaces' - self.inheritance_column = :_type_disabled - belongs_to :parent, - class_name: "#{MigrationClasses.name}::Namespace" - has_one :route, as: :source - has_many :children, - class_name: "#{MigrationClasses.name}::Namespace", - foreign_key: :parent_id - - # Overridden to have the correct `source_type` for the `route` relation - def self.name - 'Namespace' - end - - def kind - type == 'Group' ? 'group' : 'user' - end - end - - class User < ActiveRecord::Base - self.table_name = 'users' - end - - class Route < ActiveRecord::Base - self.table_name = 'routes' - belongs_to :source, polymorphic: true - end - - class Project < ActiveRecord::Base - include MigrationClasses::Routable - has_one :route, as: :source - self.table_name = 'projects' - - HASHED_STORAGE_FEATURES = { - repository: 1, - attachments: 2 - }.freeze - - def repository_storage_path - Gitlab.config.repositories.storages[repository_storage].legacy_disk_path - end - - # Overridden to have the correct `source_type` for the `route` relation - def self.name - 'Project' - end - - def hashed_storage?(feature) - raise ArgumentError, "Invalid feature" unless HASHED_STORAGE_FEATURES.include?(feature) - return false unless respond_to?(:storage_version) - - self.storage_version && self.storage_version >= HASHED_STORAGE_FEATURES[feature] - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb deleted file mode 100644 index 2c9d0d6c0d1..00000000000 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb +++ /dev/null @@ -1,196 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module RenameReservedPathsMigration - module V1 - class RenameBase - attr_reader :paths, :migration - - delegate :update_column_in_batches, - :execute, - :replace_sql, - :quote_string, - :say, - to: :migration - - def initialize(paths, migration) - @paths = paths - @migration = migration - end - - def path_patterns - @path_patterns ||= paths.flat_map { |path| ["%/#{path}", path] } - end - - def rename_path_for_routable(routable) - old_path = routable.path - old_full_path = routable.full_path - # Only remove the last occurrence of the path name to get the parent namespace path - namespace_path = remove_last_occurrence(old_full_path, old_path) - new_path = rename_path(namespace_path, old_path) - new_full_path = join_routable_path(namespace_path, new_path) - - perform_rename(routable, old_full_path, new_full_path) - - [old_full_path, new_full_path] - end - - def perform_rename(routable, old_full_path, new_full_path) - # skips callbacks & validations - new_path = new_full_path.split('/').last - routable.class.where(id: routable) - .update_all(path: new_path) - - rename_routes(old_full_path, new_full_path) - end - - def rename_routes(old_full_path, new_full_path) - routes = Route.arel_table - - quoted_old_full_path = quote_string(old_full_path) - quoted_old_wildcard_path = quote_string("#{old_full_path}/%") - - filter = - "routes.id IN "\ - "( SELECT routes.id FROM routes WHERE lower(routes.path) = lower('#{quoted_old_full_path}') "\ - "UNION SELECT routes.id FROM routes WHERE routes.path ILIKE '#{quoted_old_wildcard_path}' )" - - replace_statement = replace_sql(Route.arel_table[:path], - old_full_path, - new_full_path) - - update = Arel::UpdateManager.new - .table(routes) - .set([[routes[:path], replace_statement]]) - .where(Arel::Nodes::SqlLiteral.new(filter)) - - execute(update.to_sql) - end - - def rename_path(namespace_path, path_was) - counter = 0 - path = "#{path_was}#{counter}" - - while route_exists?(join_routable_path(namespace_path, path)) - counter += 1 - path = "#{path_was}#{counter}" - end - - path - end - - def remove_last_occurrence(string, pattern) - string.reverse.sub(pattern.reverse, "").reverse - end - - def join_routable_path(namespace_path, top_level) - if namespace_path.present? - File.join(namespace_path, top_level) - else - top_level - end - end - - def route_exists?(full_path) - MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any? - end - - def move_pages(old_path, new_path) - move_folders(pages_dir, old_path, new_path) - end - - def move_uploads(old_path, new_path) - return unless file_storage? - - move_folders(uploads_dir, old_path, new_path) - end - - def move_folders(directory, old_relative_path, new_relative_path) - old_path = File.join(directory, old_relative_path) - unless File.directory?(old_path) - say "#{old_path} doesn't exist, skipping" - return - end - - new_path = File.join(directory, new_relative_path) - FileUtils.mv(old_path, new_path) - end - - def remove_cached_html_for_projects(project_ids) - project_ids.each do |project_id| - update_column_in_batches(:projects, :description_html, nil) do |table, query| - query.where(table[:id].eq(project_id)) - end - - update_column_in_batches(:issues, :description_html, nil) do |table, query| - query.where(table[:project_id].eq(project_id)) - end - - update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| - query.where(table[:target_project_id].eq(project_id)) - end - - update_column_in_batches(:notes, :note_html, nil) do |table, query| - query.where(table[:project_id].eq(project_id)) - end - - update_column_in_batches(:milestones, :description_html, nil) do |table, query| - query.where(table[:project_id].eq(project_id)) - end - end - end - - def track_rename(type, old_path, new_path) - key = redis_key_for_type(type) - Gitlab::Redis::SharedState.with do |redis| - redis.lpush(key, [old_path, new_path].to_json) - redis.expire(key, 2.weeks.to_i) - end - say "tracked rename: #{key}: #{old_path} -> #{new_path}" - end - - def reverts_for_type(type) - key = redis_key_for_type(type) - - Gitlab::Redis::SharedState.with do |redis| - failed_reverts = [] - - while rename_info = redis.lpop(key) - path_before_rename, path_after_rename = Gitlab::Json.parse(rename_info) - say "renaming #{type} from #{path_after_rename} back to #{path_before_rename}" - begin - yield(path_before_rename, path_after_rename) - rescue StandardError => e - failed_reverts << rename_info - say "Renaming #{type} from #{path_after_rename} back to "\ - "#{path_before_rename} failed. Review the error and try "\ - "again by running the `down` action. \n"\ - "#{e.message}: \n #{e.backtrace.join("\n")}" - end - end - - failed_reverts.each { |rename_info| redis.lpush(key, rename_info) } - end - end - - def redis_key_for_type(type) - "rename:#{migration.name}:#{type}" - end - - def file_storage? - CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File - end - - def uploads_dir - File.join(CarrierWave.root, "uploads") - end - - def pages_dir - Settings.pages.path - end - end - end - end - end -end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb deleted file mode 100644 index 72ae2849911..00000000000 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb +++ /dev/null @@ -1,106 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module RenameReservedPathsMigration - module V1 - class RenameNamespaces < RenameBase - include Gitlab::ShellAdapter - - def rename_namespaces(type:) - namespaces_for_paths(type: type).each do |namespace| - rename_namespace(namespace) - end - end - - def namespaces_for_paths(type:) - namespaces = case type - when :child - MigrationClasses::Namespace.where.not(parent_id: nil) - when :top_level - MigrationClasses::Namespace.where(parent_id: nil) - end - with_paths = MigrationClasses::Route.arel_table[:path] - .matches_any(path_patterns) - namespaces.joins(:route).where(with_paths) - .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") - end - - def rename_namespace(namespace) - old_full_path, new_full_path = rename_path_for_routable(namespace) - - track_rename('namespace', old_full_path, new_full_path) - - rename_namespace_dependencies(namespace, old_full_path, new_full_path) - end - - def rename_namespace_dependencies(namespace, old_full_path, new_full_path) - move_repositories(namespace, old_full_path, new_full_path) - move_uploads(old_full_path, new_full_path) - move_pages(old_full_path, new_full_path) - rename_user(old_full_path, new_full_path) if namespace.kind == 'user' - remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id)) - end - - def revert_renames - reverts_for_type('namespace') do |path_before_rename, current_path| - matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) - namespace = MigrationClasses::Namespace.joins(:route) - .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") - .find_by(matches_path)&.becomes(MigrationClasses::Namespace) # rubocop: disable Cop/AvoidBecomes - - if namespace - perform_rename(namespace, current_path, path_before_rename) - - rename_namespace_dependencies(namespace, current_path, path_before_rename) - else - say "Couldn't rename namespace from #{current_path} back to #{path_before_rename}, "\ - "namespace was renamed, or no longer exists at the expected path" - end - end - end - - def rename_user(old_username, new_username) - MigrationClasses::User.where(username: old_username) - .update_all(username: new_username) - end - - def move_repositories(namespace, old_full_path, new_full_path) - repo_shards_for_namespace(namespace).each do |repository_storage| - # Ensure old directory exists before moving it - Gitlab::GitalyClient::NamespaceService.allow do - gitlab_shell.add_namespace(repository_storage, old_full_path) - - unless gitlab_shell.mv_namespace(repository_storage, old_full_path, new_full_path) - message = "Exception moving on shard #{repository_storage} from #{old_full_path} to #{new_full_path}" - Gitlab::AppLogger.error message - end - end - end - end - - def repo_shards_for_namespace(namespace) - projects_for_namespace(namespace).distinct.select(:repository_storage) - .map(&:repository_storage) - end - - def projects_for_namespace(namespace) - namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id]) - namespace_or_children = MigrationClasses::Project - .arel_table[:namespace_id] - .in(namespace_ids) - MigrationClasses::Project.where(namespace_or_children) - end - - def child_ids_for_parent(namespace, ids: []) - namespace.children.each do |child| - ids << child.id - child_ids_for_parent(child, ids: ids) if child.children.any? - end - ids - end - end - end - end - end -end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb deleted file mode 100644 index 155e35b64f4..00000000000 --- a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module RenameReservedPathsMigration - module V1 - class RenameProjects < RenameBase - include Gitlab::ShellAdapter - - def rename_projects - projects_for_paths.each do |project| - rename_project(project) - end - - remove_cached_html_for_projects(projects_for_paths.map(&:id)) - end - - def rename_project(project) - old_full_path, new_full_path = rename_path_for_routable(project) - - track_rename('project', old_full_path, new_full_path) - - move_project_folders(project, old_full_path, new_full_path) - end - - def move_project_folders(project, old_full_path, new_full_path) - unless project.hashed_storage?(:repository) - move_repository(project, old_full_path, new_full_path) - move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") - end - - move_uploads(old_full_path, new_full_path) unless project.hashed_storage?(:attachments) - move_pages(old_full_path, new_full_path) - end - - def revert_renames - reverts_for_type('project') do |path_before_rename, current_path| - matches_path = MigrationClasses::Route.arel_table[:path].matches(current_path) - project = MigrationClasses::Project.joins(:route) - .allow_cross_joins_across_databases(url: - 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') - .find_by(matches_path) - - if project - perform_rename(project, current_path, path_before_rename) - - move_project_folders(project, current_path, path_before_rename) - else - say "Couldn't rename project from #{current_path} back to "\ - "#{path_before_rename}, project was renamed or no longer "\ - "exists at the expected path." - - end - end - end - - def move_repository(project, old_path, new_path) - unless gitlab_shell.mv_repository(project.repository_storage, - old_path, - new_path) - Gitlab::AppLogger.error "Error moving #{old_path} to #{new_path}" - end - end - - def projects_for_paths - return @projects_for_paths if @projects_for_paths - - with_paths = MigrationClasses::Route.arel_table[:path] - .matches_any(path_patterns) - - @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths) - .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') - end - end - end - end - end -end diff --git a/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb b/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb index 4e7a4ec748b..4e3b685c06c 100644 --- a/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb +++ b/lib/gitlab/database_importers/work_items/hierarchy_restrictions_importer.rb @@ -14,17 +14,52 @@ module Gitlab ticket = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:ticket]) restrictions = [ - { parent_type_id: objective.id, child_type_id: objective.id, maximum_depth: 9 }, - { parent_type_id: objective.id, child_type_id: key_result.id, maximum_depth: 1 }, - { parent_type_id: issue.id, child_type_id: task.id, maximum_depth: 1 }, - { parent_type_id: incident.id, child_type_id: task.id, maximum_depth: 1 }, - { parent_type_id: epic.id, child_type_id: epic.id, maximum_depth: 9 }, - { parent_type_id: epic.id, child_type_id: issue.id, maximum_depth: 1 }, - { parent_type_id: ticket.id, child_type_id: task.id, maximum_depth: 1 } + { + parent_type_id: objective.id, + child_type_id: objective.id, + maximum_depth: 9, + cross_hierarchy_enabled: false + }, + { + parent_type_id: objective.id, + child_type_id: key_result.id, + maximum_depth: 1, + cross_hierarchy_enabled: false + }, + { + parent_type_id: issue.id, + child_type_id: task.id, + maximum_depth: 1, + cross_hierarchy_enabled: false + }, + { + parent_type_id: incident.id, + child_type_id: task.id, + maximum_depth: 1, + cross_hierarchy_enabled: false + }, + { + parent_type_id: epic.id, + child_type_id: epic.id, + maximum_depth: 9, + cross_hierarchy_enabled: true + }, + { + parent_type_id: epic.id, + child_type_id: issue.id, + maximum_depth: 1, + cross_hierarchy_enabled: true + }, + { + parent_type_id: ticket.id, + child_type_id: task.id, + maximum_depth: 1, + cross_hierarchy_enabled: false + } ] ::WorkItems::HierarchyRestriction.upsert_all( - restrictions, + filtered_restrictions(restrictions), unique_by: :index_work_item_hierarchy_restrictions_on_parent_and_child ) end @@ -36,6 +71,16 @@ module Gitlab Gitlab::DatabaseImporters::WorkItems::BaseTypeImporter.upsert_types ::WorkItems::Type.find_by_name_and_namespace_id(name, nil) end + + def self.filtered_restrictions(restrictions) + missing_columns = restrictions.first.keys.select do |attribute| + ::WorkItems::HierarchyRestriction.column_names.exclude?(attribute.to_s) + end + + return restrictions if missing_columns.empty? + + restrictions.map { |restriction| restriction.except(*missing_columns) } + end end end end diff --git a/lib/gitlab/database_importers/work_items/related_links_restrictions_importer.rb b/lib/gitlab/database_importers/work_items/related_links_restrictions_importer.rb new file mode 100644 index 00000000000..692764bd16d --- /dev/null +++ b/lib/gitlab/database_importers/work_items/related_links_restrictions_importer.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module WorkItems + module RelatedLinksRestrictionsImporter + # This importer populates the default link restrictions for the base work item types that support this feature. + # These rules are documented in https://docs.gitlab.com/ee/development/work_items.html#write-a-database-migration + + # rubocop:disable Metrics/AbcSize + def self.upsert_restrictions + epic = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:epic]) + issue = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:issue]) + task = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:task]) + objective = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:objective]) + key_result = find_or_create_type(::WorkItems::Type::TYPE_NAMES[:key_result]) + + restrictions = [ + # Source can relate to target and target can relate to source + { source_type_id: epic.id, target_type_id: epic.id, link_type: 0 }, + { source_type_id: epic.id, target_type_id: issue.id, link_type: 0 }, + { source_type_id: epic.id, target_type_id: task.id, link_type: 0 }, + { source_type_id: epic.id, target_type_id: objective.id, link_type: 0 }, + { source_type_id: epic.id, target_type_id: key_result.id, link_type: 0 }, + { source_type_id: issue.id, target_type_id: issue.id, link_type: 0 }, + { source_type_id: issue.id, target_type_id: task.id, link_type: 0 }, + { source_type_id: issue.id, target_type_id: objective.id, link_type: 0 }, + { source_type_id: issue.id, target_type_id: key_result.id, link_type: 0 }, + { source_type_id: task.id, target_type_id: task.id, link_type: 0 }, + { source_type_id: task.id, target_type_id: objective.id, link_type: 0 }, + { source_type_id: task.id, target_type_id: key_result.id, link_type: 0 }, + { source_type_id: objective.id, target_type_id: objective.id, link_type: 0 }, + { source_type_id: objective.id, target_type_id: key_result.id, link_type: 0 }, + { source_type_id: key_result.id, target_type_id: key_result.id, link_type: 0 }, + # Source can block target and target can be blocked by source + { source_type_id: epic.id, target_type_id: epic.id, link_type: 1 }, + { source_type_id: epic.id, target_type_id: issue.id, link_type: 1 }, + { source_type_id: epic.id, target_type_id: task.id, link_type: 1 }, + { source_type_id: epic.id, target_type_id: objective.id, link_type: 1 }, + { source_type_id: epic.id, target_type_id: key_result.id, link_type: 1 }, + { source_type_id: issue.id, target_type_id: issue.id, link_type: 1 }, + { source_type_id: issue.id, target_type_id: epic.id, link_type: 1 }, + { source_type_id: issue.id, target_type_id: task.id, link_type: 1 }, + { source_type_id: issue.id, target_type_id: objective.id, link_type: 1 }, + { source_type_id: issue.id, target_type_id: key_result.id, link_type: 1 }, + { source_type_id: task.id, target_type_id: task.id, link_type: 1 }, + { source_type_id: task.id, target_type_id: epic.id, link_type: 1 }, + { source_type_id: task.id, target_type_id: issue.id, link_type: 1 }, + { source_type_id: task.id, target_type_id: objective.id, link_type: 1 }, + { source_type_id: task.id, target_type_id: key_result.id, link_type: 1 }, + { source_type_id: objective.id, target_type_id: objective.id, link_type: 1 }, + { source_type_id: objective.id, target_type_id: key_result.id, link_type: 1 }, + { source_type_id: key_result.id, target_type_id: key_result.id, link_type: 1 }, + { source_type_id: key_result.id, target_type_id: objective.id, link_type: 1 } + ] + + ::WorkItems::RelatedLinkRestriction.upsert_all( + restrictions, + unique_by: :index_work_item_link_restrictions_on_source_link_type_target + ) + end + # rubocop:enable Metrics/AbcSize + + def self.find_or_create_type(name) + type = ::WorkItems::Type.find_by_name_and_namespace_id(name, nil) + return type if type + + Gitlab::DatabaseImporters::WorkItems::BaseTypeImporter.upsert_types + ::WorkItems::Type.find_by_name_and_namespace_id(name, nil) + end + end + end + end +end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index ff17010c11c..74bec55253f 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -3,9 +3,9 @@ module Gitlab module DependencyLinker class BaseLinker - URL_REGEX = %r{https?://[^'" ]+}.freeze - GIT_INVALID_URL_REGEX = /^git\+#{URL_REGEX}/.freeze - REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze + URL_REGEX = %r{https?://[^'" ]+} + GIT_INVALID_URL_REGEX = /^git\+#{URL_REGEX}/ + REPO_REGEX = %r{[^/'" ]+/[^/'" ]+} class_attribute :file_type diff --git a/lib/gitlab/dependency_linker/gemfile_linker.rb b/lib/gitlab/dependency_linker/gemfile_linker.rb index c6e02248b0a..baba6511d62 100644 --- a/lib/gitlab/dependency_linker/gemfile_linker.rb +++ b/lib/gitlab/dependency_linker/gemfile_linker.rb @@ -8,8 +8,8 @@ module Gitlab self.package_keyword = :gem self.file_type = :gemfile - GITHUB_REGEX = /(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/.freeze - GIT_REGEX = /(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/.freeze + GITHUB_REGEX = /(github:|:github\s*=>)\s*['"](?<name>[^'"]+)['"]/ + GIT_REGEX = /(git:|:git\s*=>)\s*['"](?<name>#{URL_REGEX})['"]/ private diff --git a/lib/gitlab/dependency_linker/godeps_json_linker.rb b/lib/gitlab/dependency_linker/godeps_json_linker.rb index 049a807b760..1d12198d637 100644 --- a/lib/gitlab/dependency_linker/godeps_json_linker.rb +++ b/lib/gitlab/dependency_linker/godeps_json_linker.rb @@ -3,7 +3,7 @@ module Gitlab module DependencyLinker class GodepsJsonLinker < JsonLinker - NESTED_REPO_REGEX = %r{([^/]+/)+[^/]+?}.freeze + NESTED_REPO_REGEX = %r{([^/]+/)+[^/]+?} self.file_type = :godeps_json diff --git a/lib/gitlab/dependency_linker/podspec_linker.rb b/lib/gitlab/dependency_linker/podspec_linker.rb index f6da17efff4..55149877d15 100644 --- a/lib/gitlab/dependency_linker/podspec_linker.rb +++ b/lib/gitlab/dependency_linker/podspec_linker.rb @@ -5,7 +5,7 @@ module Gitlab class PodspecLinker < MethodLinker include Cocoapods - STRING_REGEX = /['"](?<name>[^'"]+)['"]/.freeze + STRING_REGEX = /['"](?<name>[^'"]+)['"]/ self.file_type = :podspec diff --git a/lib/gitlab/deploy_key_access.rb b/lib/gitlab/deploy_key_access.rb index a582c978be7..caf2a73c338 100644 --- a/lib/gitlab/deploy_key_access.rb +++ b/lib/gitlab/deploy_key_access.rb @@ -16,17 +16,6 @@ module Gitlab attr_reader :deploy_key - def protected_tag_accessible_to?(ref, action:) - if Feature.enabled?(:deploy_key_for_protected_tags, project) - super - else - assert_project! - # a deploy key can always push a protected tag - # (which is not always the case when pushing to a protected branch) - true - end - end - def can_collaborate?(_ref) assert_project! diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d5c0b187f92..de7be6efd72 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -7,7 +7,7 @@ module Gitlab attr_reader :diff, :repository, :diff_refs, :fallback_diff_refs, :unique_identifier - delegate :new_file?, :deleted_file?, :renamed_file?, + delegate :new_file?, :deleted_file?, :renamed_file?, :unidiff, :old_path, :new_path, :a_mode, :b_mode, :mode_changed?, :submodule?, :expanded?, :too_large?, :collapsed?, :line_count, :has_binary_notice?, to: :diff, prefix: false diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 95ea3fe9f0f..d2524ae1761 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,7 +3,7 @@ module Gitlab module Diff class Highlight - PREFIX_REGEXP = /\A(.)/.freeze + PREFIX_REGEXP = /\A(.)/ attr_reader :diff_file, :diff_lines, :repository, :project diff --git a/lib/gitlab/diff/pair_selector.rb b/lib/gitlab/diff/pair_selector.rb index 2e5ee3a7363..e848f5107ae 100644 --- a/lib/gitlab/diff/pair_selector.rb +++ b/lib/gitlab/diff/pair_selector.rb @@ -20,7 +20,7 @@ module Gitlab # Runs end at the end of the string (the last line) or before a space (for an unchanged line) (?=\s|\z) - }x.freeze + }x # rubocop: enable Lint/MixedRegexpCaptureTypes def initialize(lines) diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index a8c0108fa34..e847d05ae71 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -59,8 +59,11 @@ module Gitlab end def compare(start_sha, head_sha, straight: false) + include_stats = !Feature.enabled?(:remove_request_stats_for_tracing, project) + compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) - compare.diffs(paths: paths, expanded: true, ignore_whitespace_change: @ignore_whitespace_change) + compare.diffs(paths: paths, expanded: true, ignore_whitespace_change: @ignore_whitespace_change, include_stats: + include_stats) end end end diff --git a/lib/gitlab/diff/suggestions_parser.rb b/lib/gitlab/diff/suggestions_parser.rb index 6f126147113..85f7a5c7527 100644 --- a/lib/gitlab/diff/suggestions_parser.rb +++ b/lib/gitlab/diff/suggestions_parser.rb @@ -4,7 +4,7 @@ module Gitlab module Diff class SuggestionsParser # Matches for instance "-1", "+1" or "-1+2". - SUGGESTION_CONTEXT = /^(\-(?<above>\d+))?(\+(?<below>\d+))?$/.freeze + SUGGESTION_CONTEXT = /^(\-(?<above>\d+))?(\+(?<below>\d+))?$/ CSS = 'pre.language-suggestion' XPATH = Gitlab::Utils::Nokogiri.css_to_xpath(CSS).freeze diff --git a/lib/gitlab/doctor/reset_tokens.rb b/lib/gitlab/doctor/reset_tokens.rb new file mode 100644 index 00000000000..45333e2effb --- /dev/null +++ b/lib/gitlab/doctor/reset_tokens.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Doctor + class ResetTokens + attr_reader :logger + + PRINT_PROGRESS_EVERY = 1000 + + def initialize(logger, model_names:, token_names:, dry_run: true) + @logger = logger + @model_names = model_names + @token_names = token_names + @dry_run = dry_run + end + + def run! + logger.info "Resetting #{@token_names.join(', ')} on #{@model_names.join(', ')} if they can not be read" + logger.info "Executing in DRY RUN mode, no records will actually be updated" if @dry_run + Rails.application.eager_load! + + models_with_encrypted_tokens.each do |model| + fix_model(model) + end + logger.info "Done!" + end + + private + + def fix_model(model) + matched_token_names = @token_names & model.encrypted_token_authenticatable_fields.map(&:to_s) + + return if matched_token_names.empty? + + total_count = model.count + + model.find_each.with_index do |instance, index| + matched_token_names.each do |attribute_name| + fix_attribute(instance, attribute_name) + end + + logger.info "Checked #{index + 1}/#{total_count} #{model.name.pluralize}" if index % PRINT_PROGRESS_EVERY == 0 + end + logger.info "Checked #{total_count} #{model.name.pluralize}" + end + + def fix_attribute(instance, attribute_name) + instance.public_send(attribute_name) # rubocop:disable GitlabSecurity/PublicSend + rescue OpenSSL::Cipher::CipherError, TypeError + logger.debug "> Fix #{instance.class.name}[#{instance.id}].#{attribute_name}" + instance.public_send("reset_#{attribute_name}!") unless @dry_run # rubocop:disable GitlabSecurity/PublicSend + rescue StandardError => e + logger.debug( + "> Something went wrong for #{instance.class.name}[#{instance.id}].#{attribute_name}: #{e}".color(:red)) + + false + end + + def models_with_encrypted_tokens + ApplicationRecord.descendants.select do |model| + @model_names.include?(model.name) && model.include?(TokenAuthenticatable) + end + end + end + end +end diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 2e487c42cb5..19826138075 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -6,7 +6,7 @@ module Gitlab class BaseHandler attr_reader :mail, :mail_key - HANDLER_ACTION_BASE_REGEX ||= /(?<project_slug>.+)-(?<project_id>\d+)/.freeze + HANDLER_ACTION_BASE_REGEX ||= /(?<project_slug>.+)-(?<project_id>\d+)/ def initialize(mail, mail_key) @mail = mail diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 869bcc6e2be..cac6c29f10b 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -11,8 +11,8 @@ module Gitlab class CreateIssueHandler < BaseHandler include ReplyProcessing - HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue\z/.freeze - HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+(?<incoming_email_token>.*)\z/.freeze + HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue\z/ + HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+(?<incoming_email_token>.*)\z/ def initialize(mail, mail_key) super(mail, mail_key) diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index c723c2762c7..6e25202241c 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -12,8 +12,8 @@ module Gitlab class CreateMergeRequestHandler < BaseHandler include ReplyProcessing - HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-merge-request\z/.freeze - HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+merge-request\+(?<incoming_email_token>.*)/.freeze + HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-merge-request\z/ + HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\+merge-request\+(?<incoming_email_token>.*)/ def initialize(mail, mail_key) super(mail, mail_key) diff --git a/lib/gitlab/email/handler/create_note_on_issuable_handler.rb b/lib/gitlab/email/handler/create_note_on_issuable_handler.rb index aed3647744a..8fea0593c78 100644 --- a/lib/gitlab/email/handler/create_note_on_issuable_handler.rb +++ b/lib/gitlab/email/handler/create_note_on_issuable_handler.rb @@ -15,7 +15,7 @@ module Gitlab attr_reader :issuable_iid - HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue-(?<issuable_iid>\d+)\z/.freeze + HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue-(?<issuable_iid>\d+)\z/ def initialize(mail, mail_key) super(mail, mail_key) diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 949fa554aeb..ebc4e9c2c8c 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -10,9 +10,9 @@ module Gitlab include ReplyProcessing include Gitlab::Utils::StrongMemoize - HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-issue-\z/.freeze - HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\z/.freeze - PROJECT_KEY_PATTERN = /\A(?<slug>.+)-(?<key>[a-z0-9_]+)\z/.freeze + HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-issue-\z/ + HANDLER_REGEX_LEGACY = /\A(?<project_path>[^\+]*)\z/ + PROJECT_KEY_PATTERN = /\A(?<slug>.+)-(?<key>[a-z0-9_]+)\z/ def initialize(mail, mail_key, service_desk_key: nil) if service_desk_key @@ -75,9 +75,10 @@ module Gitlab def contains_custom_email_address_verification_subaddress? return false unless Feature.enabled?(:service_desk_custom_email, project) + return false unless to_address.present? # Verification email only has one recipient - mail.to.first.include?(ServiceDeskSetting::CUSTOM_EMAIL_VERIFICATION_SUBADDRESS) + to_address.include?(ServiceDeskSetting::CUSTOM_EMAIL_VERIFICATION_SUBADDRESS) end def handled_custom_email_address_verification? @@ -209,6 +210,11 @@ module Gitlab (mail.reply_to || []).first || mail.from.first || mail.sender end + def to_address + mail.to&.first + end + strong_memoize_attr :to_address + def can_handle_legacy_format? project_path && project_path.include?('/') && !mail_key.include?('+') end diff --git a/lib/gitlab/email/message/build_ios_app_guide.rb b/lib/gitlab/email/message/build_ios_app_guide.rb deleted file mode 100644 index 4acf558a6a2..00000000000 --- a/lib/gitlab/email/message/build_ios_app_guide.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - class BuildIosAppGuide - include Gitlab::Email::Message::InProductMarketing::Helper - include Gitlab::Routing - - attr_accessor :format - - def initialize(format: :html) - @format = format - end - - def subject_line - s_('InProductMarketing|Get set up to build for iOS') - end - - def title - s_("InProductMarketing|Building for iOS? We've got you covered.") - end - - def body_line1 - s_( - 'InProductMarketing|Want to get your iOS app up and running, including publishing all the way to ' \ - 'TestFlight? Follow our guide to set up GitLab and fastlane to publish iOS apps to the App Store.' - ) - end - - def cta_text - s_('InProductMarketing|Learn how to build for iOS') - end - - def cta_link - action_link(cta_text, 'https://about.gitlab.com/blog/2019/03/06/ios-publishing-with-gitlab-and-fastlane/') - end - - def cta2_text - s_('InProductMarketing|Watch iOS building in action.') - end - - def cta2_link - action_link(cta2_text, 'https://www.youtube.com/watch?v=325FyJt7ZG8') - end - - def logo_path - 'mailers/in_product_marketing/create-0.png' - end - - def unsubscribe - unsubscribe_message - end - end - end - end -end diff --git a/lib/gitlab/email/message/in_product_marketing/helper.rb b/lib/gitlab/email/message/in_product_marketing/helper.rb deleted file mode 100644 index 0770e5f4d76..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/helper.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - module Helper - include ActionView::Context - include ActionView::Helpers::TagHelper - - def footer_links - links = [ - [s_('InProductMarketing|Blog'), 'https://about.gitlab.com/blog'], - [s_('InProductMarketing|Twitter'), 'https://twitter.com/gitlab'], - [s_('InProductMarketing|Facebook'), 'https://www.facebook.com/gitlab'], - [s_('InProductMarketing|YouTube'), 'https://www.youtube.com/channel/UCnMGQ8QHMAnVIsI3xJrihhg'] - ] - case format - when :html - links.map do |text, link| - ActionController::Base.helpers.link_to(text, link) - end - else - '| ' + links.map do |text, link| - [text, link].join(' ') - end.join("\n| ") - end - end - - def address - s_('InProductMarketing|%{strong_start}GitLab Inc.%{strong_end} 268 Bush Street, #350, San Francisco, CA 94104, USA').html_safe % strong_options - end - - def unsubscribe_message(self_managed_preferences_link = nil) - parts = Gitlab.com? ? unsubscribe_com : unsubscribe_self_managed(self_managed_preferences_link) - - case format - when :html - parts.join(' ') - else - parts.join("\n" + ' ' * 16) - end - end - - private - - def unsubscribe_link - unsubscribe_url = Gitlab.com? ? '%tag_unsubscribe_url%' : profile_notifications_url - - link(s_('InProductMarketing|unsubscribe'), unsubscribe_url) - end - - def unsubscribe_com - [ - s_('InProductMarketing|If you no longer wish to receive marketing emails from us,'), - s_('InProductMarketing|you may %{unsubscribe_link} at any time.') % { unsubscribe_link: unsubscribe_link } - ] - end - - def unsubscribe_self_managed(preferences_link) - [ - s_('InProductMarketing|To opt out of these onboarding emails, %{unsubscribe_link}.') % { unsubscribe_link: unsubscribe_link }, - s_("InProductMarketing|If you don't want to receive marketing emails directly from GitLab, %{marketing_preference_link}.") % { marketing_preference_link: preferences_link } - ] - end - - def strong_options - case format - when :html - { strong_start: '<b>'.html_safe, strong_end: '</b>'.html_safe } - else - { strong_start: '', strong_end: '' } - end - end - - def link(text, link) - case format - when :html - ActionController::Base.helpers.link_to text, link - else - "#{text} (#{link})" - end - end - - def action_link(text, link) - case format - when :html - ActionController::Base.helpers.link_to text, link, target: '_blank', rel: 'noopener noreferrer' - else - [text, link].join(' >> ') - end - end - end - end - end - end -end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index ee11105537b..d5877234c3a 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -8,7 +8,7 @@ module Gitlab class Receiver include Gitlab::Utils::StrongMemoize - RECEIVED_HEADER_REGEX = /for\s+\<([^<]+)\>/.freeze + RECEIVED_HEADER_REGEX = /for\s+\<([^<]+)\>/ # Errors that are purely from users and not anything we can control USER_ERRORS = [ @@ -50,6 +50,7 @@ module Gitlab delivered_to: delivered_to.map(&:value), envelope_to: envelope_to.map(&:value), x_envelope_to: x_envelope_to.map(&:value), + cc_address: cc, # reduced down to what looks like an email in the received headers received_recipients: recipients_from_received_headers, meta: { @@ -84,23 +85,27 @@ module Gitlab def mail_key strong_memoize(:mail_key) do - key_from_to_header || key_from_additional_headers + find_first_key_from(to) || key_from_additional_headers end end - def key_from_to_header - to.find do |address| - key = email_class.key_from_address(address) - break key if key + def find_first_key_from(items) + items.each do |item| + email = item.is_a?(Mail::Field) ? item.value : item + + key = email_class.key_from_address(email) + return key if key end + nil end def key_from_additional_headers find_key_from_references || - find_key_from_delivered_to_header || - find_key_from_envelope_to_header || - find_key_from_x_envelope_to_header || - find_first_key_from_received_headers + find_first_key_from(delivered_to) || + find_first_key_from(envelope_to) || + find_first_key_from(x_envelope_to) || + find_first_key_from(recipients_from_received_headers) || + find_first_key_from(cc) end def ensure_references_array(references) @@ -131,6 +136,10 @@ module Gitlab Array(mail.to) end + def cc + Array(mail.cc) + end + def delivered_to Array(mail[:delivered_to]) end @@ -147,34 +156,6 @@ module Gitlab Array(mail[:received]) end - def find_key_from_delivered_to_header - delivered_to.find do |header| - key = email_class.key_from_address(header.value) - break key if key - end - end - - def find_key_from_envelope_to_header - envelope_to.find do |header| - key = email_class.key_from_address(header.value) - break key if key - end - end - - def find_key_from_x_envelope_to_header - x_envelope_to.find do |header| - key = email_class.key_from_address(header.value) - break key if key - end - end - - def find_first_key_from_received_headers - recipients_from_received_headers.find do |email| - key = email_class.key_from_address(email) - break key if key - end - end - def recipients_from_received_headers strong_memoize :emails_from_received_headers do received.filter_map { |header| header.value[RECEIVED_HEADER_REGEX, 1] } diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index 99240f2ad48..b080cb197d4 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -152,8 +152,6 @@ module Gitlab message.delete_prefix(BOM_UTF8) end - private - def force_encode_utf8(message) raise ArgumentError unless message.respond_to?(:force_encoding) return message if message.encoding == Encoding::UTF_8 && message.valid_encoding? @@ -163,6 +161,8 @@ module Gitlab message.force_encoding("UTF-8") end + private + # Escapes \x80 - \xFF characters not supported by UTF-8 def escape_chars(char) bytes = char.bytes diff --git a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb index 398ddebd355..3b0b4c6e935 100644 --- a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb +++ b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb @@ -177,7 +177,7 @@ module Gitlab ErrorRepository::Pagination.new(pagination_hash['next'], pagination_hash['prev']) end - LINK_PATTERN = %r{cursor=(?<cursor>[^&]+).*; rel="(?<direction>\w+)"}.freeze + LINK_PATTERN = %r{cursor=(?<cursor>[^&]+).*; rel="(?<direction>\w+)"} def parse_pagination_link(content) match = LINK_PATTERN.match(content) diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index 8679f17eb9b..e887e455792 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -12,8 +12,6 @@ module Gitlab # ExclusiveLease. # class ExclusiveLease - include Gitlab::Utils::StrongMemoize - PREFIX = 'gitlab:exclusive_lease' NoKey = Class.new(ArgumentError) @@ -33,7 +31,7 @@ module Gitlab EOS def self.get_uuid(key) - with_read_redis do |redis| + Gitlab::Redis::ClusterSharedState.with do |redis| redis.get(redis_shared_state_key(key)) || false end end @@ -63,7 +61,7 @@ module Gitlab def self.cancel(key, uuid) return unless key.present? - with_write_redis do |redis| + Gitlab::Redis::ClusterSharedState.with do |redis| redis.eval(LUA_CANCEL_SCRIPT, keys: [ensure_prefixed_key(key)], argv: [uuid]) end end @@ -81,12 +79,6 @@ module Gitlab # Removes any existing exclusive_lease from redis # Don't run this in a live system without making sure no one is using the leases def self.reset_all!(scope = '*') - Gitlab::Redis::SharedState.with do |redis| - redis.scan_each(match: redis_shared_state_key(scope)).each do |key| - redis.del(key) - end - end - Gitlab::Redis::ClusterSharedState.with do |redis| redis.scan_each(match: redis_shared_state_key(scope)).each do |key| redis.del(key) @@ -94,15 +86,6 @@ module Gitlab end end - def self.use_cluster_shared_state? - Gitlab::SafeRequestStore[:use_cluster_shared_state] ||= - Feature.enabled?(:use_cluster_shared_state_for_exclusive_lease) - end - - def self.use_double_lock? - Gitlab::SafeRequestStore[:use_double_lock] ||= Feature.enabled?(:enable_exclusive_lease_double_lock_rw) - end - def initialize(key, uuid: nil, timeout:) @redis_shared_state_key = self.class.redis_shared_state_key(key) @timeout = timeout @@ -112,23 +95,10 @@ module Gitlab # Try to obtain the lease. Return lease UUID on success, # false if the lease is already taken. def try_obtain - return try_obtain_with_new_lock if self.class.use_cluster_shared_state? - # Performing a single SET is atomic - obtained = set_lease(Gitlab::Redis::SharedState) && @uuid - - # traffic to new store is minimal since only the first lock holder can run SETNX in ClusterSharedState - return false unless obtained - return obtained unless self.class.use_double_lock? - return obtained if same_store # 2nd setnx will surely fail if store are the same - - second_lock_obtained = set_lease(Gitlab::Redis::ClusterSharedState) && @uuid - - # cancel is safe since it deletes key only if value matches uuid - # i.e. it will not delete the held lock on ClusterSharedState - cancel unless second_lock_obtained - - second_lock_obtained + Gitlab::Redis::ClusterSharedState.with do |redis| + redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid + end end # This lease is waiting to obtain @@ -139,7 +109,7 @@ module Gitlab # Try to renew an existing lease. Return lease UUID on success, # false if the lease is taken by a different UUID or inexistent. def renew - self.class.with_write_redis do |redis| + Gitlab::Redis::ClusterSharedState.with do |redis| result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout]) result == @uuid end @@ -147,7 +117,7 @@ module Gitlab # Returns true if the key for this lease is set. def exists? - self.class.with_read_redis do |redis| + Gitlab::Redis::ClusterSharedState.with do |redis| redis.exists?(@redis_shared_state_key) # rubocop:disable CodeReuse/ActiveRecord end end @@ -156,66 +126,17 @@ module Gitlab # # This method will return `nil` if no TTL could be obtained. def ttl - self.class.with_read_redis do |redis| + Gitlab::Redis::ClusterSharedState.with do |redis| ttl = redis.ttl(@redis_shared_state_key) ttl if ttl > 0 end end - # rubocop:disable CodeReuse/ActiveRecord - def self.with_write_redis(&blk) - if use_cluster_shared_state? - result = Gitlab::Redis::ClusterSharedState.with(&blk) - Gitlab::Redis::SharedState.with(&blk) - - result - elsif use_double_lock? - result = Gitlab::Redis::SharedState.with(&blk) - Gitlab::Redis::ClusterSharedState.with(&blk) - - result - else - Gitlab::Redis::SharedState.with(&blk) - end - end - - def self.with_read_redis(&blk) - if use_cluster_shared_state? - Gitlab::Redis::ClusterSharedState.with(&blk) - elsif use_double_lock? - Gitlab::Redis::SharedState.with(&blk) || Gitlab::Redis::ClusterSharedState.with(&blk) - else - Gitlab::Redis::SharedState.with(&blk) - end - end - # rubocop:enable CodeReuse/ActiveRecord - # Gives up this lease, allowing it to be obtained by others. def cancel self.class.cancel(@redis_shared_state_key, @uuid) end - - private - - def set_lease(redis_class) - redis_class.with do |redis| - redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) - end - end - - def try_obtain_with_new_lock - # checks shared-state to avoid 2 versions of the application acquiring 1 lock - # wait for held lock to expire or yielded in case any process on old version is running - return false if Gitlab::Redis::SharedState.with { |c| c.exists?(@redis_shared_state_key) } # rubocop:disable CodeReuse/ActiveRecord - - set_lease(Gitlab::Redis::ClusterSharedState) && @uuid - end - - def same_store - Gitlab::Redis::ClusterSharedState.with(&:id) == Gitlab::Redis::SharedState.with(&:id) # rubocop:disable CodeReuse/ActiveRecord - end - strong_memoize_attr :same_store end end diff --git a/lib/gitlab/exclusive_lease_helpers.rb b/lib/gitlab/exclusive_lease_helpers.rb index 7cf0232fbf2..8836cb34745 100644 --- a/lib/gitlab/exclusive_lease_helpers.rb +++ b/lib/gitlab/exclusive_lease_helpers.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Gitlab - # This module provides helper methods which are intregrated with GitLab::ExclusiveLease + # This module provides helper methods which are integrated with GitLab::ExclusiveLease module ExclusiveLeaseHelpers FailedToObtainLockError = Class.new(StandardError) diff --git a/lib/gitlab/experiment/rollout/feature.rb b/lib/gitlab/experiment/rollout/feature.rb index bf31dfe08a0..4ff61aa3551 100644 --- a/lib/gitlab/experiment/rollout/feature.rb +++ b/lib/gitlab/experiment/rollout/feature.rb @@ -27,7 +27,8 @@ module Gitlab # # If the `Feature.enabled?` check is false, we return nil implicitly, # which will assign the control. Otherwise we call super, which will - # assign a variant evenly, or based on our provided distribution rules. + # assign a variant based on our provided distribution rules. + # Otherwise we will assign a variant evenly across the behaviours without control. def execute_assignment super if ::Feature.enabled?(feature_flag_name, self, type: :experiment) end @@ -67,6 +68,10 @@ module Gitlab def feature_flag_name experiment.name.tr('/', '_') end + + def behavior_names + super - [:control] + end end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 9150213020e..37f593ed551 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -8,7 +8,7 @@ module Gitlab # https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904' BLANK_SHA = ('0' * 40).freeze - COMMIT_ID = /\A#{Gitlab::Git::Commit::RAW_FULL_SHA_PATTERN}\z/.freeze + COMMIT_ID = /\A#{Gitlab::Git::Commit::RAW_FULL_SHA_PATTERN}\z/ TAG_REF_PREFIX = "refs/tags/" BRANCH_REF_PREFIX = "refs/heads/" diff --git a/lib/gitlab/git/base_error.rb b/lib/gitlab/git/base_error.rb index 330e947844c..48235b30d90 100644 --- a/lib/gitlab/git/base_error.rb +++ b/lib/gitlab/git/base_error.rb @@ -5,7 +5,7 @@ module Gitlab module Git class BaseError < StandardError METADATA_KEY = :gitaly_error_metadata - DEBUG_ERROR_STRING_REGEX = /(.*?) debug_error_string:.*$/m.freeze + DEBUG_ERROR_STRING_REGEX = /(.*?) debug_error_string:.*$/m GRPC_CODES = { '0' => 'ok', '1' => 'cancelled', diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 21d2eaec041..3d2bde6f0a7 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -4,6 +4,7 @@ module Gitlab module Git class Blame include Gitlab::EncodingHelper + include Gitlab::Git::WrapsGitalyErrors attr_reader :lines, :blames, :range @@ -29,13 +30,19 @@ module Gitlab end def load_blame - output = encode_utf8( - @repo.gitaly_commit_client.raw_blame(@sha, @path, range: range_spec) - ) - + output = encode_utf8(fetch_raw_blame) process_raw_blame(output) end + def fetch_raw_blame + wrapped_gitaly_errors do + @repo.gitaly_commit_client.raw_blame(@sha, @path, range: range_spec) + end + # Return empty result when blame range is out-of-range + rescue ArgumentError + "" + end + def process_raw_blame(output) start_line = nil lines = [] diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index de25fa7e099..743bac62764 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -33,7 +33,7 @@ module Gitlab SERIALIZE_KEYS = %i[diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large].freeze - BINARY_NOTICE_PATTERN = %r{Binary files a\/(.*) and b\/(.*) differ}.freeze + BINARY_NOTICE_PATTERN = %r{Binary files (.*) and (.*) differ} class << self def between(repo, head, base, options = {}, *paths) @@ -183,6 +183,16 @@ module Gitlab a_mode == '160000' || b_mode == '160000' end + def unidiff + return diff if diff.blank? + return json_safe_diff if detect_binary?(@diff) || has_binary_notice? + + old_path_header = new_file? ? '/dev/null' : "a/#{old_path}" + new_path_header = deleted_file? ? '/dev/null' : "b/#{new_path}" + + "--- #{old_path_header}\n+++ #{new_path_header}\n" + diff + end + def line_count @line_count ||= Util.count_lines(@diff) end diff --git a/lib/gitlab/git/pre_receive_error.rb b/lib/gitlab/git/pre_receive_error.rb index b84ac656927..49116775215 100644 --- a/lib/gitlab/git/pre_receive_error.rb +++ b/lib/gitlab/git/pre_receive_error.rb @@ -14,7 +14,7 @@ module Gitlab 'GL-HOOK-ERR:' # Messages marked as safe by user ].freeze - SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/.freeze + SAFE_MESSAGE_REGEX = /^(#{SAFE_MESSAGE_PREFIXES.join('|')})\s*(?<safe_message>.+)/ attr_reader :raw_message diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index dfbf8292f54..a98cf95edf4 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -163,12 +163,6 @@ module Gitlab branch_names.count end - def rename(new_relative_path) - wrapped_gitaly_errors do - gitaly_repository_client.rename(new_relative_path) - end - end - def remove wrapped_gitaly_errors do gitaly_repository_client.remove @@ -261,12 +255,8 @@ module Gitlab def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:, path: nil) ref ||= root_ref - if Feature.enabled?(:resolve_ambiguous_archives, container) - commit_id = extract_commit_id_from_ref(ref) - return {} if commit_id.nil? - else - commit_id = ref - end + commit_id = extract_commit_id_from_ref(ref) + return {} if commit_id.nil? commit = Gitlab::Git::Commit.find(self, commit_id) return {} if commit.nil? @@ -1220,6 +1210,14 @@ module Gitlab end end + def get_file_attributes(revision, file_paths, attributes) + wrapped_gitaly_errors do + gitaly_repository_client + .get_file_attributes(revision, file_paths, attributes) + .attribute_infos + end + end + private def repository_info_size_megabytes diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb index 632b4133f2e..57cced97d02 100644 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -4,15 +4,8 @@ module Gitlab module Git module RuggedImpl module UseRugged - def use_rugged?(repo, feature_key) - return Feature.enabled?(feature_key) if Feature.persisted_name?(feature_key) - - # Disable Rugged auto-detect(can_use_disk?) when Puma threads>1 - # https://gitlab.com/gitlab-org/gitlab/issues/119326 - return false if running_puma_with_multiple_threads? - return false if Feature.enabled?(:skip_rugged_auto_detect, type: :ops) - - Gitlab::GitalyClient.can_use_disk?(repo.storage) + def use_rugged?(_, _) + false end def execute_rugged_call(method_name, *args) @@ -49,9 +42,7 @@ module Gitlab end def rugged_enabled_through_feature_flag? - rugged_feature_keys.any? do |feature_key| - Feature.enabled?(feature_key) - end + false end end end diff --git a/lib/gitlab/git_audit_event.rb b/lib/gitlab/git_audit_event.rb new file mode 100644 index 00000000000..b8365bdf41f --- /dev/null +++ b/lib/gitlab/git_audit_event.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + class GitAuditEvent # rubocop:disable Gitlab/NamespacedClass + attr_reader :project, :user, :author + + def initialize(player, project) + @project = project + @author = player.is_a?(::API::Support::GitAccessActor) ? player.deploy_key_or_user : player + @user = player.is_a?(::API::Support::GitAccessActor) ? player.user : player + end + + def send_audit_event(msg) + return if user.blank? || project.blank? + + audit_context = { + name: 'repository_git_operation', + stream_only: true, + author: author, + scope: project, + target: project, + message: msg + } + + ::Gitlab::Audit::Auditor.audit(audit_context) + end + end +end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index f2ea6f17d90..5ec58fc4f44 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -56,13 +56,6 @@ module Gitlab # https://gitlab.com/gitlab-org/gitaly/-/blob/bf9f52bc/client/dial.go#L78 'grpc.keepalive_time_ms': 20000, 'grpc.keepalive_permit_without_calls': 1, - # Enable client-side automatic retry. After enabled, gRPC requests will be retried when there are connectivity - # problems with the target host. Only transparent failures, which mean requests fail before leaving clients, are - # eligible. Other cases are configurable via retry policy in service config (below). In theory, we can auto-retry - # read-only RPCs. Gitaly defines a custom field in service proto. Unfortunately, gRPC ruby doesn't support - # descriptor reflection. - # For more information please visit https://github.com/grpc/proposal/blob/master/A6-client-retries.md - 'grpc.enable_retries': 1, # Service config is a mechanism for grpc to control the behavior of gRPC client. It defines the client-side # balancing strategy and retry policy. The config receives a raw JSON string. The format is defined here: # https://github.com/grpc/grpc-proto/blob/master/grpc/service_config/service_config.proto @@ -72,7 +65,119 @@ module Gitlab # grpc creates multiple subchannels to all targets retrurned by the resolver. Requests are distributed to # those subchannels in a round-robin fashion. # More about client-side load-balancing: https://gitlab.com/groups/gitlab-org/-/epics/8971#note_1207008162 - "loadBalancingConfig": [{ "round_robin": {} }] + "loadBalancingConfig": [{ "round_robin": {} }], + # Enable retries for read-only RPCs. With this setting the client to will resend requests that fail with + # the following conditions: + # 1. An `UNAVAILABLE` status code was received. + # 2. No response-headers were received from the server. + # This allows the client to handle momentary service interruptions without user-facing errors. gRPC's + # automatic 'transparent retries' may also be sent. + # For more information please visit https://github.com/grpc/proposal/blob/master/A6-client-retries.md + 'methodConfig': [ + { + # Gitaly sets an `op_type` `MethodOption` on RPCs to note if it mutates a repository. We cannot + # programatically detect read-only RPCs, i.e. those safe to retry, because Ruby's protobuf + # implementation does not provide access to `MethodOptions`. That feature is being tracked under + # https://github.com/protocolbuffers/protobuf/issues/1198. When that is complete we can replace this + # table. + 'name': [ + { 'service': 'gitaly.BlobService', 'method': 'GetBlob' }, + { 'service': 'gitaly.BlobService', 'method': 'GetBlobs' }, + { 'service': 'gitaly.BlobService', 'method': 'GetLFSPointers' }, + { 'service': 'gitaly.BlobService', 'method': 'GetAllLFSPointers' }, + { 'service': 'gitaly.BlobService', 'method': 'ListAllBlobs' }, + { 'service': 'gitaly.BlobService', 'method': 'ListAllLFSPointers' }, + { 'service': 'gitaly.BlobService', 'method': 'ListBlobs' }, + { 'service': 'gitaly.BlobService', 'method': 'ListLFSPointers' }, + { 'service': 'gitaly.CommitService', 'method': 'CheckObjectsExist' }, + { 'service': 'gitaly.CommitService', 'method': 'CommitIsAncestor' }, + { 'service': 'gitaly.CommitService', 'method': 'CommitLanguages' }, + { 'service': 'gitaly.CommitService', 'method': 'CommitStats' }, + { 'service': 'gitaly.CommitService', 'method': 'CommitsByMessage' }, + { 'service': 'gitaly.CommitService', 'method': 'CountCommits' }, + { 'service': 'gitaly.CommitService', 'method': 'CountDivergingCommits' }, + { 'service': 'gitaly.CommitService', 'method': 'FilterShasWithSignatures' }, + { 'service': 'gitaly.CommitService', 'method': 'FindAllCommits' }, + { 'service': 'gitaly.CommitService', 'method': 'FindCommit' }, + { 'service': 'gitaly.CommitService', 'method': 'FindCommits' }, + { 'service': 'gitaly.CommitService', 'method': 'GetCommitMessages' }, + { 'service': 'gitaly.CommitService', 'method': 'GetCommitSignatures' }, + { 'service': 'gitaly.CommitService', 'method': 'GetTreeEntries' }, + { 'service': 'gitaly.CommitService', 'method': 'LastCommitForPath' }, + { 'service': 'gitaly.CommitService', 'method': 'ListAllCommits' }, + { 'service': 'gitaly.CommitService', 'method': 'ListCommits' }, + { 'service': 'gitaly.CommitService', 'method': 'ListCommitsByOid' }, + { 'service': 'gitaly.CommitService', 'method': 'ListCommitsByRefName' }, + { 'service': 'gitaly.CommitService', 'method': 'ListFiles' }, + { 'service': 'gitaly.CommitService', 'method': 'ListLastCommitsForTree' }, + { 'service': 'gitaly.CommitService', 'method': 'RawBlame' }, + { 'service': 'gitaly.CommitService', 'method': 'TreeEntry' }, + { 'service': 'gitaly.ConflictsService', 'method': 'ListConflictFiles' }, + { 'service': 'gitaly.DiffService', 'method': 'CommitDelta' }, + { 'service': 'gitaly.DiffService', 'method': 'CommitDiff' }, + { 'service': 'gitaly.DiffService', 'method': 'DiffStats' }, + { 'service': 'gitaly.DiffService', 'method': 'FindChangedPaths' }, + { 'service': 'gitaly.DiffService', 'method': 'GetPatchID' }, + { 'service': 'gitaly.DiffService', 'method': 'RawDiff' }, + { 'service': 'gitaly.DiffService', 'method': 'RawPatch' }, + { 'service': 'gitaly.ObjectPoolService', 'method': 'GetObjectPool' }, + { 'service': 'gitaly.RefService', 'method': 'FindAllBranches' }, + { 'service': 'gitaly.RefService', 'method': 'FindAllRemoteBranches' }, + { 'service': 'gitaly.RefService', 'method': 'FindAllTags' }, + { 'service': 'gitaly.RefService', 'method': 'FindBranch' }, + { 'service': 'gitaly.RefService', 'method': 'FindDefaultBranchName' }, + { 'service': 'gitaly.RefService', 'method': 'FindLocalBranches' }, + { 'service': 'gitaly.RefService', 'method': 'FindRefsByOID' }, + { 'service': 'gitaly.RefService', 'method': 'FindTag' }, + { 'service': 'gitaly.RefService', 'method': 'GetTagMessages' }, + { 'service': 'gitaly.RefService', 'method': 'GetTagSignatures' }, + { 'service': 'gitaly.RefService', 'method': 'ListBranchNamesContainingCommit' }, + { 'service': 'gitaly.RefService', 'method': 'ListRefs' }, + { 'service': 'gitaly.RefService', 'method': 'ListTagNamesContainingCommit' }, + { 'service': 'gitaly.RefService', 'method': 'RefExists' }, + { 'service': 'gitaly.RemoteService', 'method': 'FindRemoteRepository' }, + { 'service': 'gitaly.RemoteService', 'method': 'FindRemoteRootRef' }, + { 'service': 'gitaly.RemoteService', 'method': 'UpdateRemoteMirror' }, + { 'service': 'gitaly.RepositoryService', 'method': 'BackupCustomHooks' }, + { 'service': 'gitaly.RepositoryService', 'method': 'BackupRepository' }, + { 'service': 'gitaly.RepositoryService', 'method': 'CalculateChecksum' }, + { 'service': 'gitaly.RepositoryService', 'method': 'CreateBundle' }, + { 'service': 'gitaly.RepositoryService', 'method': 'Fsck' }, + { 'service': 'gitaly.RepositoryService', 'method': 'FindLicense' }, + { 'service': 'gitaly.RepositoryService', 'method': 'FindMergeBase' }, + { 'service': 'gitaly.RepositoryService', 'method': 'FullPath' }, + { 'service': 'gitaly.RepositoryService', 'method': 'HasLocalBranches' }, + { 'service': 'gitaly.RepositoryService', 'method': 'GetArchive' }, + { 'service': 'gitaly.RepositoryService', 'method': 'GetConfig' }, + { 'service': 'gitaly.RepositoryService', 'method': 'GetCustomHooks' }, + { 'service': 'gitaly.RepositoryService', 'method': 'GetFileAttributes' }, + { 'service': 'gitaly.RepositoryService', 'method': 'GetInfoAttributes' }, + { 'service': 'gitaly.RepositoryService', 'method': 'GetObject' }, + { 'service': 'gitaly.RepositoryService', 'method': 'GetObjectDirectorySize' }, + { 'service': 'gitaly.RepositoryService', 'method': 'GetRawChanges' }, + { 'service': 'gitaly.RepositoryService', 'method': 'GetSnapshot' }, + { 'service': 'gitaly.RepositoryService', 'method': 'ObjectSize' }, + { 'service': 'gitaly.RepositoryService', 'method': 'ObjectFormat' }, + { 'service': 'gitaly.RepositoryService', 'method': 'RepositoryExists' }, + { 'service': 'gitaly.RepositoryService', 'method': 'RepositoryInfo' }, + { 'service': 'gitaly.RepositoryService', 'method': 'RepositorySize' }, + { 'service': 'gitaly.RepositoryService', 'method': 'SearchFilesByContent' }, + { 'service': 'gitaly.RepositoryService', 'method': 'SearchFilesByName' }, + { 'service': 'gitaly.ServerService', 'method': 'ClockSynced' }, + { 'service': 'gitaly.ServerService', 'method': 'DiskStatistics' }, + { 'service': 'gitaly.ServerService', 'method': 'ReadinessCheck' }, + { 'service': 'gitaly.ServerService', 'method': 'ServerInfo' }, + { 'service': 'grpc.health.v1.Health', 'method': 'Check' } + ], + 'retryPolicy': { + 'maxAttempts': 3, # Initial request, plus up to two retries. + 'initialBackoff': '0.25s', + 'maxBackoff': '1s', + 'backoffMultiplier': 2, # Minimum retry duration is 750ms. + 'retryableStatusCodes': ['UNAVAILABLE'] + } + } + ] }.to_json } end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 573e3547202..1ef5b0f96c2 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -418,9 +418,6 @@ module Gitlab response = gitaly_client_call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) response.reduce([]) { |memo, msg| memo << msg.data }.join - # Temporary fix, use structured errors when they are available: https://gitlab.com/gitlab-org/gitaly/-/issues/5594 - rescue GRPC::Internal - "" end def find_commit(revision) diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb deleted file mode 100644 index 05aee2fa55d..00000000000 --- a/lib/gitlab/gitaly_client/namespace_service.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GitalyClient - class NamespaceService - extend Gitlab::TemporarilyAllow - - NamespaceServiceAccessError = Class.new(StandardError) - ALLOW_KEY = :allow_namespace - - def self.allow - temporarily_allow(ALLOW_KEY) { yield } - end - - def self.denied? - !temporarily_allowed?(ALLOW_KEY) - end - - def initialize(storage) - raise NamespaceServiceAccessError if self.class.denied? - - @storage = storage - end - - def add(name) - request = Gitaly::AddNamespaceRequest.new(storage_name: @storage, name: name) - - gitaly_client_call(:add_namespace, request, timeout: GitalyClient.fast_timeout) - end - - def remove(name) - request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name) - - gitaly_client_call(:remove_namespace, request, timeout: GitalyClient.long_timeout) - end - - def rename(from, to) - request = Gitaly::RenameNamespaceRequest.new(storage_name: @storage, from: from, to: to) - - gitaly_client_call(:rename_namespace, request, timeout: GitalyClient.fast_timeout) - end - - def exists?(name) - request = Gitaly::NamespaceExistsRequest.new(storage_name: @storage, name: name) - - response = gitaly_client_call(:namespace_exists, request, timeout: GitalyClient.fast_timeout) - response.exists - end - - private - - def gitaly_client_call(type, request, timeout: nil) - GitalyClient.call(@storage, :namespace_service, type, request, timeout: timeout) - end - end - end -end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 9ea541e083d..d92bf5263f1 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -319,12 +319,6 @@ module Gitlab gitaly_client_call(@storage, :object_pool_service, :disconnect_git_alternates, request, timeout: GitalyClient.long_timeout) end - def rename(relative_path) - request = Gitaly::RenameRepositoryRequest.new(repository: @gitaly_repo, relative_path: relative_path) - - gitaly_client_call(@storage, :repository_service, :rename_repository, request, timeout: GitalyClient.fast_timeout) - end - def remove request = Gitaly::RemoveRepositoryRequest.new(repository: @gitaly_repo) @@ -359,6 +353,13 @@ module Gitlab ) end + def get_file_attributes(revision, paths, attributes) + request = Gitaly::GetFileAttributesRequest + .new(repository: @gitaly_repo, revision: revision, paths: paths, attributes: attributes) + + gitaly_client_call(@repository.storage, :repository_service, :get_file_attributes, request, timeout: GitalyClient.fast_timeout) + end + private def search_results_from_response(gitaly_response, options = {}) diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb index d16f4d7587b..47080ea1979 100644 --- a/lib/gitlab/github_import/bulk_importing.rb +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -32,7 +32,7 @@ module Gitlab log_error(github_identifiers, build_record.errors.full_messages) errors << { validation_errors: build_record.errors, - github_identifiers: github_identifiers + external_identifiers: github_identifiers } next end @@ -69,7 +69,7 @@ module Gitlab correlation_id_value: correlation_id_value, retry_count: nil, created_at: Time.zone.now, - external_identifiers: error[:github_identifiers] + external_identifiers: error[:external_identifiers] } end @@ -79,8 +79,7 @@ module Gitlab private def log_and_increment_counter(value, operation) - Gitlab::Import::Logger.info( - import_type: :github, + Logger.info( project_id: project.id, importer: self.class.name, message: "#{value} #{object_type.to_s.pluralize} #{operation}" @@ -95,12 +94,11 @@ module Gitlab end def log_error(github_identifiers, messages) - Gitlab::Import::Logger.error( - import_type: :github, + Logger.error( project_id: project.id, importer: self.class.name, message: messages, - github_identifiers: github_identifiers + external_identifiers: github_identifiers ) end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 23d4faa3dde..5a0ae680ab8 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -284,10 +284,10 @@ module Gitlab def on_retry proc do |exception, try, elapsed_time, next_interval| - Gitlab::Import::Logger.info( + Logger.info( message: "GitHub connection retry triggered", 'error.class': exception.class, - 'error.message': exception.message, + 'exception.message': exception.message, try_count: try, elapsed_time_s: elapsed_time, wait_to_retry_s: next_interval diff --git a/lib/gitlab/github_import/clients/proxy.rb b/lib/gitlab/github_import/clients/proxy.rb index 27030f5382a..a95a8cddc8d 100644 --- a/lib/gitlab/github_import/clients/proxy.rb +++ b/lib/gitlab/github_import/clients/proxy.rb @@ -10,19 +10,15 @@ module Gitlab REPOS_COUNT_CACHE_KEY = 'github-importer/provider-repo-count/%{type}/%{user_id}' - def initialize(access_token, client_options) - @client = pick_client(access_token, client_options) + def initialize(access_token) + @client = Gitlab::GithubImport::Client.new(access_token) end def repos(search_text, options) - return { repos: filtered(client.repos, search_text) } if use_legacy? - fetch_repos_via_graphql(search_text, options) end def count_repos_by(relation_type, user_id) - return if use_legacy? - key = format(REPOS_COUNT_CACHE_KEY, type: relation_type, user_id: user_id) ::Gitlab::Cache::Import::Caching.read_integer(key, timeout: 5.minutes) || @@ -40,22 +36,6 @@ module Gitlab } end - def pick_client(access_token, client_options) - return Gitlab::GithubImport::Client.new(access_token) unless use_legacy? - - Gitlab::LegacyGithubImport::Client.new(access_token, **client_options) - end - - def filtered(collection, search_text) - return collection if search_text.blank? - - collection.select { |item| item[:name].to_s.downcase.include?(search_text) } - end - - def use_legacy? - Feature.disabled?(:remove_legacy_github_client) - end - def fetch_and_cache_repos_count_via_graphql(relation_type, key) response = client.count_repos_by_relation_type_graphql(relation_type: relation_type) count = response.dig(:data, :search, :repositoryCount) diff --git a/lib/gitlab/github_import/exceptions.rb b/lib/gitlab/github_import/exceptions.rb index 3a36b64a11b..b7d93182603 100644 --- a/lib/gitlab/github_import/exceptions.rb +++ b/lib/gitlab/github_import/exceptions.rb @@ -6,6 +6,8 @@ module Gitlab # Sometimes it's not clear which of not implemented interfaces caused this error. # We need custom exception to be able to add text that gives extra context. NotImplementedError = Class.new(StandardError) + + NoteableNotFound = Class.new(StandardError) end end end diff --git a/lib/gitlab/github_import/importer/attachments/issues_importer.rb b/lib/gitlab/github_import/importer/attachments/issues_importer.rb index c8f0b59fd18..a0e1a3f2d25 100644 --- a/lib/gitlab/github_import/importer/attachments/issues_importer.rb +++ b/lib/gitlab/github_import/importer/attachments/issues_importer.rb @@ -24,7 +24,7 @@ module Gitlab private def collection - project.issues.select(:id, :description, :iid) + project.issues.id_not_in(already_imported_ids).select(:id, :description, :iid) end def ordering_column diff --git a/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb index cd3a327a846..22b3e7c640b 100644 --- a/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb +++ b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb @@ -24,7 +24,7 @@ module Gitlab private def collection - project.merge_requests.select(:id, :description, :iid) + project.merge_requests.id_not_in(already_imported_ids).select(:id, :description, :iid) end def ordering_column diff --git a/lib/gitlab/github_import/importer/attachments/notes_importer.rb b/lib/gitlab/github_import/importer/attachments/notes_importer.rb index aa38a7a3a3f..5ab0cf5b6b0 100644 --- a/lib/gitlab/github_import/importer/attachments/notes_importer.rb +++ b/lib/gitlab/github_import/importer/attachments/notes_importer.rb @@ -26,7 +26,7 @@ module Gitlab # TODO: exclude :system, :noteable_type from select after removing override Note#note method # https://gitlab.com/gitlab-org/gitlab/-/issues/369923 def collection - project.notes.user.select(:id, :note, :system, :noteable_type) + project.notes.id_not_in(already_imported_ids).user.select(:id, :note, :system, :noteable_type) end end end diff --git a/lib/gitlab/github_import/importer/attachments/releases_importer.rb b/lib/gitlab/github_import/importer/attachments/releases_importer.rb index 7d6dbeb901e..0527170f5e1 100644 --- a/lib/gitlab/github_import/importer/attachments/releases_importer.rb +++ b/lib/gitlab/github_import/importer/attachments/releases_importer.rb @@ -24,7 +24,7 @@ module Gitlab private def collection - project.releases.select(:id, :description, :tag) + project.releases.id_not_in(already_imported_ids).select(:id, :description, :tag) end end end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 44ffcd7a1e4..d49180e6927 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -36,13 +36,6 @@ module Gitlab Logger.warn(message: e.message, 'error.class': e.class.name) import_with_legacy_diff_note - rescue ActiveRecord::InvalidForeignKey => e - # It's possible the project and the issue have been deleted since - # scheduling this job. In this case we'll just skip creating the note - Logger.info( - message: e.message, - github_identifiers: note.github_identifiers - ) end private @@ -71,6 +64,7 @@ module Gitlab discussion_id: note.discussion_id, noteable_id: merge_request_id, project_id: project.id, + namespace_id: project.project_namespace_id, author_id: author_id, note: note_body, commit_id: note.original_commit_id, @@ -132,7 +126,7 @@ module Gitlab Logger.info( project_id: project.id, importer: self.class.name, - github_identifiers: note.github_identifiers, + external_identifiers: note.github_identifiers, model: model ) end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index a537841ecf3..3cf67d7df96 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -29,8 +29,8 @@ module Gitlab def execute Issue.transaction do if (issue_id = create_issue) - create_assignees(issue_id) issuable_finder.cache_database_id(issue_id) + create_assignees(issue_id) update_search_data(issue_id) end end @@ -64,9 +64,6 @@ module Gitlab issue.validate! insert_and_return_id(attributes, project.issues) - rescue ActiveRecord::InvalidForeignKey - # It's possible the project has been deleted since scheduling this - # job. In this case we'll just skip creating the issue. end # Stores all issue assignees in the database. diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 04da015a33f..dc5c6e49b55 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -17,7 +17,9 @@ module Gitlab end def execute - return unless (noteable_id = find_noteable_id) + noteable_id = find_noteable_id + + raise Exceptions::NoteableNotFound, 'Error to find noteable_id for note' unless noteable_id author_id, author_found = user_finder.author_id_for(note) @@ -25,6 +27,7 @@ module Gitlab noteable_type: note.noteable_type, noteable_id: noteable_id, project_id: project.id, + namespace_id: project.project_namespace_id, author_id: author_id, note: note_body(author_found), discussion_id: note.discussion_id, @@ -33,19 +36,15 @@ module Gitlab updated_at: note.updated_at } - note = Note.new(attributes.merge(importing: true)) - note.validate! + Note.new(attributes.merge(importing: true)).validate! - # We're using bulk_insert here so we can bypass any validations and - # callbacks. Running these would result in a lot of unnecessary SQL + # We're using bulk_insert here so we can bypass any callbacks. + # Running these would result in a lot of unnecessary SQL # queries being executed when importing large projects. # Note: if you're going to replace `legacy_bulk_insert` with something that trigger callback # to generate HTML version - you also need to regenerate it in # Gitlab::GithubImport::Importer::NoteAttachmentsImporter. ApplicationRecord.legacy_bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert - rescue ActiveRecord::InvalidForeignKey - # It's possible the project and the issue have been deleted since - # scheduling this job. In this case we'll just skip creating the note. end # Returns the ID of the issue or merge request to create the note for. diff --git a/lib/gitlab/github_import/importer/pull_request_importer.rb b/lib/gitlab/github_import/importer/pull_request_importer.rb index 5690a2cc997..acdafef670c 100644 --- a/lib/gitlab/github_import/importer/pull_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_importer.rb @@ -27,9 +27,9 @@ module Gitlab mr, already_exists = create_merge_request if mr + issuable_finder.cache_database_id(mr.id) set_merge_request_assignees(mr) insert_git_data(mr, already_exists) - issuable_finder.cache_database_id(mr.id) end end diff --git a/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb index 0a92aee801d..7f78df615a2 100644 --- a/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb @@ -19,6 +19,9 @@ module Gitlab review_requests = client.pull_request_review_requests(repo, merge_request.iid) review_requests[:merge_request_id] = merge_request.id review_requests[:merge_request_iid] = merge_request.iid + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + yield review_requests mark_merge_request_imported(merge_request) @@ -42,6 +45,10 @@ module Gitlab :pull_request_review_requests end + def object_type + :pull_request_review_request + end + # rubocop:disable CodeReuse/ActiveRecord def merge_request_collection project.merge_requests diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 62863ba67fd..671e023e90b 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -44,7 +44,7 @@ module Gitlab pname = project.path_with_namespace - Gitlab::Import::Logger.info( + Logger.info( message: 'GitHub importer finished updating repository', project_name: pname ) diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index cfc1ec526b0..cccd99f48b1 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -211,6 +211,12 @@ module Gitlab private + # Returns the set used to track "already imported" objects. + # Items are the values returned by `#id_for_already_imported_cache`. + def already_imported_ids + Gitlab::Cache::Import::Caching.values_from_set(already_imported_cache_key) + end + def additional_object_data {} end diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index 191e15962a6..e8e515d1f87 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -7,7 +7,7 @@ module Gitlab include ToHash include ExposeAttribute - NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i.freeze + NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i expose_attribute :noteable_id, :commit_id, :file_path, :diff_hunk, :author, :created_at, :updated_at, diff --git a/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb b/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb index 38b560f21c0..db36e81c5b8 100644 --- a/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb +++ b/lib/gitlab/github_import/representation/diff_notes/discussion_id.rb @@ -6,7 +6,7 @@ module Gitlab module DiffNotes class DiscussionId NOTEABLE_TYPE = 'MergeRequest' - DISCUSSION_CACHE_REGEX = %r{/(?<repo>[^/]*)/pull/(?<iid>\d+)}i.freeze + DISCUSSION_CACHE_REGEX = %r{/(?<repo>[^/]*)/pull/(?<iid>\d+)}i DISCUSSION_CACHE_KEY = 'github-importer/discussion-id-map/%{project}/%{noteable_id}/%{original_note_id}' def initialize(note) diff --git a/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb b/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb index 38b15c4b5bb..fdf74fd9c9f 100644 --- a/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb +++ b/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb @@ -16,7 +16,7 @@ module Gitlab # - the ```suggestion tag must be the first text of the line # - it might have up to 3 spaces before the ```suggestion tag # - extra text on the ```suggestion tag line will be ignored - GITHUB_SUGGESTION = /^\ {,3}(?<suggestion>```suggestion\b).*(?<eol>\R)/.freeze + GITHUB_SUGGESTION = /^\ {,3}(?<suggestion>```suggestion\b).*(?<eol>\R)/ def initialize(note:, start_line: nil, end_line: nil) @note = note diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index 7a8bdfb1c64..76adbb651af 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -12,7 +12,7 @@ module Gitlab expose_attribute :noteable_id, :noteable_type, :author, :note, :created_at, :updated_at, :note_id - NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i.freeze + NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i # Builds a note from a GitHub API response. # diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb index 73a5f49a9e3..a4170f4147f 100644 --- a/lib/gitlab/github_import/settings.rb +++ b/lib/gitlab/github_import/settings.rb @@ -61,8 +61,11 @@ module Gitlab additional_access_tokens: user_settings[:additional_access_tokens] ) - import_data = project.create_or_update_import_data( - data: { optional_stages: optional_stages }, + import_data = project.build_or_assign_import_data( + data: { + optional_stages: optional_stages, + timeout_strategy: user_settings[:timeout_strategy] + }, credentials: credentials ) diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 1832f071a44..4bf2d8a0aca 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -271,8 +271,7 @@ module Gitlab end def log(message, username: nil) - Gitlab::Import::Logger.info( - import_type: :github, + Logger.info( project_id: project.id, class: self.class.name, username: username, diff --git a/lib/gitlab/golang.rb b/lib/gitlab/golang.rb index 1b625a3a514..f0afe493019 100644 --- a/lib/gitlab/golang.rb +++ b/lib/gitlab/golang.rb @@ -32,7 +32,7 @@ module Gitlab # vX.Y.(Z+1)-0.yyyymmddhhmmss-sha1337beef0, when most recent prior tag is vX.Y.Z if version.minor != 0 || version.patch != 0 - m = /\A(.*\.)?0\./.freeze.match pre + m = /\A(.*\.)?0\./.match pre return false unless m pre = pre[m[0].length..] @@ -40,7 +40,7 @@ module Gitlab # This pattern is intentionally more forgiving than the patterns # above. Correctness is verified by #validate_pseudo_version. - /\A\d{14}-\h+\z/.freeze.match? pre + /\A\d{14}-\h+\z/.match? pre end def parse_pseudo_version(semver) diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index eefa23142af..e057b4bb6f1 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -7,15 +7,16 @@ module Gitlab include WebpackHelper def add_gon_variables - gon.api_version = 'v4' - gon.default_avatar_url = default_avatar_url - gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size - gon.asset_host = ActionController::Base.asset_host - gon.webpack_public_path = webpack_public_path - gon.relative_url_root = Gitlab.config.gitlab.relative_url_root - gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class - gon.markdown_surround_selection = current_user&.markdown_surround_selection - gon.markdown_automatic_lists = current_user&.markdown_automatic_lists + gon.api_version = 'v4' + gon.default_avatar_url = default_avatar_url + gon.max_file_size = Gitlab::CurrentSettings.max_attachment_size + gon.asset_host = ActionController::Base.asset_host + gon.webpack_public_path = webpack_public_path + gon.relative_url_root = Gitlab.config.gitlab.relative_url_root + gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class + gon.markdown_surround_selection = current_user&.markdown_surround_selection + gon.markdown_automatic_lists = current_user&.markdown_automatic_lists + gon.math_rendering_limits_enabled = Gitlab::CurrentSettings.math_rendering_limits_enabled add_browsersdk_tracking @@ -75,10 +76,8 @@ module Gitlab push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) push_frontend_feature_flag(:unbatch_graphql_queries, current_user) - push_frontend_feature_flag(:server_side_frecent_namespaces, current_user) # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248 push_frontend_feature_flag(:remove_monitor_metrics) - push_frontend_feature_flag(:gitlab_duo, current_user) push_frontend_feature_flag(:custom_emoji) end @@ -121,7 +120,9 @@ module Gitlab end def add_browsersdk_tracking - return unless Gitlab.com? && Feature.enabled?(:browsersdk_tracking) + return unless Gitlab.com? && Feature.enabled?(:browsersdk_tracking) && Feature.enabled?(:gl_analytics_tracking, +Feature.current_request) + return if ENV['GITLAB_ANALYTICS_URL'].blank? || ENV['GITLAB_ANALYTICS_ID'].blank? gon.analytics_url = ENV['GITLAB_ANALYTICS_URL'] diff --git a/lib/gitlab/graphql/authorize/connection_filter_extension.rb b/lib/gitlab/graphql/authorize/connection_filter_extension.rb index 889c024ab5e..9b6398fc498 100644 --- a/lib/gitlab/graphql/authorize/connection_filter_extension.rb +++ b/lib/gitlab/graphql/authorize/connection_filter_extension.rb @@ -46,12 +46,15 @@ module Gitlab end def after_resolve(value:, context:, **rest) - return value if value.is_a?(GraphQL::Execution::Execute::Skip) + return value if value.is_a?(GraphQL::Execution::Skip) if @field.connection? redact_connection(value, context) elsif @field.type.list? - redact_list(value.to_a, context) unless value.nil? + unless value.nil? + value = value.to_a + redact_list(value, context) + end end value diff --git a/lib/gitlab/graphql/deprecations.rb b/lib/gitlab/graphql/deprecations.rb index 221b19bf8a3..61a49bd7473 100644 --- a/lib/gitlab/graphql/deprecations.rb +++ b/lib/gitlab/graphql/deprecations.rb @@ -11,6 +11,14 @@ module Gitlab attr_accessor :deprecation end + def initialize(*args, **kwargs, &block) + init_gitlab_deprecation(kwargs) + + super + + update_deprecation_description + end + def visible?(ctx) super && ctx[:remove_deprecated] == true ? deprecation.nil? : true end @@ -37,7 +45,12 @@ module Gitlab end kwargs[:deprecation_reason] = deprecation.deprecation_reason - kwargs[:description] = deprecation.edit_description(kwargs[:description]) + end + + def update_deprecation_description + return if deprecation.nil? + + description(deprecation.edit_description(description)) end end end diff --git a/lib/gitlab/graphql/deprecations/deprecation.rb b/lib/gitlab/graphql/deprecations/deprecation.rb index 0cf555b0e34..6821c7b0ab9 100644 --- a/lib/gitlab/graphql/deprecations/deprecation.rb +++ b/lib/gitlab/graphql/deprecations/deprecation.rb @@ -74,10 +74,10 @@ module Gitlab end def edit_description(original_description) - @original_description = original_description - return unless original_description + @original_description = original_description&.strip + return unless @original_description - original_description + description_suffix + @original_description + description_suffix end def original_description diff --git a/lib/gitlab/graphql/pagination/active_record_array_connection.rb b/lib/gitlab/graphql/pagination/active_record_array_connection.rb index 9e40f79b2fd..ce16693cf89 100644 --- a/lib/gitlab/graphql/pagination/active_record_array_connection.rb +++ b/lib/gitlab/graphql/pagination/active_record_array_connection.rb @@ -59,6 +59,7 @@ module Gitlab def dup self.class.new( items.dup, + context: context, first: first, after: after, max_page_size: max_page_size, diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb index 9cdc84ffaa3..fc569bdc5dc 100644 --- a/lib/gitlab/graphql/queries.rb +++ b/lib/gitlab/graphql/queries.rb @@ -5,14 +5,14 @@ require 'find' module Gitlab module Graphql module Queries - IMPORT_RE = /^#\s*import "(?<path>[^"]+)"$/m.freeze - EE_ELSE_CE = /^ee_else_ce/.freeze - HOME_RE = /^~/.freeze - HOME_EE = %r{^ee/}.freeze - DOTS_RE = %r{^(\.\./)+}.freeze - DOT_RE = %r{^\./}.freeze - IMPLICIT_ROOT = %r{^app/}.freeze - CONN_DIRECTIVE = /@connection\(key: "\w+"\)/.freeze + IMPORT_RE = /^#\s*import "(?<path>[^"]+)"$/m + EE_ELSE_CE = /^ee_else_ce/ + HOME_RE = /^~/ + HOME_EE = %r{^ee/} + DOTS_RE = %r{^(\.\./)+} + DOT_RE = %r{^\./} + IMPLICIT_ROOT = %r{^app/} + CONN_DIRECTIVE = /@connection\(key: "\w+"\)/ class WrappedError delegate :message, to: :@error diff --git a/lib/gitlab/harbor/query.rb b/lib/gitlab/harbor/query.rb index fc0ac539e07..2d03412fde2 100644 --- a/lib/gitlab/harbor/query.rb +++ b/lib/gitlab/harbor/query.rb @@ -8,7 +8,7 @@ module Gitlab attr_reader :client, :repository_id, :artifact_id, :search, :limit, :sort, :page DEFAULT_LIMIT = 10 - SORT_REGEX = %r{\A(creation_time|update_time|name) (asc|desc)\z}.freeze + SORT_REGEX = %r{\A(creation_time|update_time|name) (asc|desc)\z} validates :page, numericality: { greater_than: 0, integer: true }, allow_blank: true validates :limit, numericality: { greater_than: 0, less_than_or_equal_to: 25, integer: true }, allow_blank: true diff --git a/lib/gitlab/hashed_storage/migrator.rb b/lib/gitlab/hashed_storage/migrator.rb deleted file mode 100644 index 912e2ee99e9..00000000000 --- a/lib/gitlab/hashed_storage/migrator.rb +++ /dev/null @@ -1,125 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HashedStorage - # Hashed Storage Migrator - # - # This is responsible for scheduling and flagging projects - # to be migrated from Legacy to Hashed storage, either one by one or in bulk. - class Migrator - BATCH_SIZE = 100 - - # Schedule a range of projects to be bulk migrated with #bulk_migrate asynchronously - # - # @param [Integer] start first project id for the range - # @param [Integer] finish last project id for the range - def bulk_schedule_migration(start:, finish:) - ::HashedStorage::MigratorWorker.perform_async(start, finish) - end - - # Schedule a range of projects to be bulk rolledback with #bulk_rollback asynchronously - # - # @param [Integer] start first project id for the range - # @param [Integer] finish last project id for the range - def bulk_schedule_rollback(start:, finish:) - ::HashedStorage::RollbackerWorker.perform_async(start, finish) - end - - # Start migration of projects from specified range - # - # Flagging a project to be migrated is a synchronous action - # but the migration runs through async jobs - # - # @param [Integer] start first project id for the range - # @param [Integer] finish last project id for the range - # rubocop: disable CodeReuse/ActiveRecord - def bulk_migrate(start:, finish:) - projects = build_relation(start, finish) - - projects.with_route.find_each(batch_size: BATCH_SIZE) do |project| - migrate(project) - end - end - # rubocop: enable CodeReuse/ActiveRecord - - # Start rollback of projects from specified range - # - # Flagging a project to be rolled back is a synchronous action - # but the rollback runs through async jobs - # - # @param [Integer] start first project id for the range - # @param [Integer] finish last project id for the range - # rubocop: disable CodeReuse/ActiveRecord - def bulk_rollback(start:, finish:) - projects = build_relation(start, finish) - - projects.with_route.find_each(batch_size: BATCH_SIZE) do |project| - rollback(project) - end - end - # rubocop: enable CodeReuse/ActiveRecord - - # Flag a project to be migrated to Hashed Storage - # - # @param [Project] project that will be migrated - def migrate(project) - Gitlab::AppLogger.info "Starting storage migration of #{project.full_path} (ID=#{project.id})..." - - project.migrate_to_hashed_storage! - rescue StandardError => err - Gitlab::AppLogger.error("#{err.message} migrating storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") - end - - # Flag a project to be rolled-back to Legacy Storage - # - # @param [Project] project that will be rolled-back - def rollback(project) - Gitlab::AppLogger.info "Starting storage rollback of #{project.full_path} (ID=#{project.id})..." - - project.rollback_to_legacy_storage! - rescue StandardError => err - Gitlab::AppLogger.error("#{err.message} rolling-back storage of #{project.full_path} (ID=#{project.id}), trace - #{err.backtrace}") - end - - # Returns whether we have any pending storage migration - # - def migration_pending? - any_non_empty_queue?(::HashedStorage::MigratorWorker, ::HashedStorage::ProjectMigrateWorker) - end - - # Returns whether we have any pending storage rollback - # - def rollback_pending? - any_non_empty_queue?(::HashedStorage::RollbackerWorker, ::HashedStorage::ProjectRollbackWorker) - end - - # Remove all remaining scheduled rollback operations - # - def abort_rollback! - [::HashedStorage::RollbackerWorker, ::HashedStorage::ProjectRollbackWorker].each do |worker| - Sidekiq::Queue.new(worker.queue).clear - end - end - - private - - def any_non_empty_queue?(*workers) - workers.any? do |worker| - Sidekiq::Queue.new(worker.queue).size != 0 # rubocop:disable Style/ZeroLengthPredicate - end - end - - # rubocop: disable CodeReuse/ActiveRecord - def build_relation(start, finish) - relation = Project - table = Project.arel_table - - relation = relation.where(table[:id].gteq(start)) if start - relation = relation.where(table[:id].lteq(finish)) if finish - - relation - end - # rubocop: enable CodeReuse/ActiveRecord - end - end -end diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb deleted file mode 100644 index d3468569e5e..00000000000 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ /dev/null @@ -1,129 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HashedStorage - module RakeHelper - def self.batch_size - ENV.fetch('BATCH', 200).to_i - end - - def self.listing_limit - ENV.fetch('LIMIT', 500).to_i - end - - def self.range_from - ENV['ID_FROM'] - end - - def self.range_to - ENV['ID_TO'] - end - - def self.using_ranges? - !range_from.nil? && !range_to.nil? - end - - def self.range_single_item? - using_ranges? && range_from == range_to - end - - # rubocop: disable CodeReuse/ActiveRecord - def self.project_id_batches_migration(&block) - Project.with_unmigrated_storage.in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches - ids = relation.pluck(:id) - - yield ids.min, ids.max - end - end - # rubocop: enable CodeReuse/ActiveRecord - - # rubocop: disable CodeReuse/ActiveRecord - def self.project_id_batches_rollback(&block) - Project.with_storage_feature(:repository).in_batches(of: batch_size, start: range_from, finish: range_to) do |relation| # rubocop: disable Cop/InBatches - ids = relation.pluck(:id) - - yield ids.min, ids.max - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def self.legacy_attachments_relation - Upload.inner_join_local_uploads_projects.merge(Project.without_storage_feature(:attachments)) - end - - def self.hashed_attachments_relation - Upload.inner_join_local_uploads_projects.merge(Project.with_storage_feature(:attachments)) - end - - def self.relation_summary(relation_name, relation) - relation_count = relation.count - $stdout.puts "* Found #{relation_count} #{relation_name}".color(:green) - - relation_count - end - - def self.projects_list(relation_name, relation) - listing(relation_name, relation.with_route) do |project| - $stdout.puts " - #{project.full_path} (id: #{project.id})".color(:red) - $stdout.puts " #{project.repository.disk_path}" - end - end - - def self.attachments_list(relation_name, relation) - listing(relation_name, relation) do |upload| - $stdout.puts " - #{upload.path} (id: #{upload.id})".color(:red) - end - end - - # rubocop: disable CodeReuse/ActiveRecord - def self.listing(relation_name, relation) - relation_count = relation_summary(relation_name, relation) - return unless relation_count > 0 - - limit = listing_limit - - if relation_count > limit - $stdout.puts " ! Displaying first #{limit} #{relation_name}..." - end - - relation.find_each(batch_size: batch_size).with_index do |element, index| - yield element - - break if index + 1 >= limit - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def self.prune(relation_name, relation, dry_run: true, root: nil) - root ||= '../repositories' - - known_paths = Set.new - listing(relation_name, relation) { |p| known_paths << "#{root}/#{p.repository.disk_path}" } - - marked_for_deletion = Set.new(Dir["#{root}/@hashed/*/*/*"]) - marked_for_deletion.reject! do |path| - base = path.gsub(/\.(\w+\.)?git$/, '') - known_paths.include?(base) - end - - if marked_for_deletion.empty? - $stdout.puts "No orphaned directories found. Nothing to do!" - else - n = marked_for_deletion.size - $stdout.puts "Found #{n} orphaned #{'directory'.pluralize(n)}" - $stdout.puts "Dry run. (Run again with FORCE=1 to delete). We would have deleted:" if dry_run - end - - marked_for_deletion.each do |p| - p = Pathname.new(p) - if dry_run - $stdout.puts " - #{p}" - else - $stdout.puts "Removing #{p}" - p.rmtree - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/puma_check.rb b/lib/gitlab/health_checks/puma_check.rb index 2dc8a093572..efe3d65db91 100644 --- a/lib/gitlab/health_checks/puma_check.rb +++ b/lib/gitlab/health_checks/puma_check.rb @@ -20,7 +20,7 @@ module Gitlab def check return unless Gitlab::Runtime.puma? - stats = Puma.stats + stats = ::Puma.stats stats = Gitlab::Json.parse(stats) # If `workers` is missing this means that diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index feb54fcca0c..9c9816e142e 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -10,21 +10,14 @@ require_relative 'http_connection_adapter' module Gitlab class HTTP - BlockedUrlError = Class.new(StandardError) - RedirectionTooDeep = Class.new(StandardError) - ReadTotalTimeout = Class.new(Net::ReadTimeout) - HeaderReadTimeout = Class.new(Net::ReadTimeout) - SilentModeBlockedError = Class.new(StandardError) + BlockedUrlError = Gitlab::HTTP_V2::BlockedUrlError + RedirectionTooDeep = Gitlab::HTTP_V2::RedirectionTooDeep + ReadTotalTimeout = Gitlab::HTTP_V2::ReadTotalTimeout + HeaderReadTimeout = Gitlab::HTTP_V2::HeaderReadTimeout + SilentModeBlockedError = Gitlab::HTTP_V2::SilentModeBlockedError - HTTP_TIMEOUT_ERRORS = [ - Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Gitlab::HTTP::ReadTotalTimeout - ].freeze - HTTP_ERRORS = HTTP_TIMEOUT_ERRORS + [ - EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, - Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, - Gitlab::HTTP::BlockedUrlError, Gitlab::HTTP::RedirectionTooDeep, - Net::HTTPBadResponse - ].freeze + HTTP_TIMEOUT_ERRORS = Gitlab::HTTP_V2::HTTP_TIMEOUT_ERRORS + HTTP_ERRORS = Gitlab::HTTP_V2::HTTP_ERRORS DEFAULT_TIMEOUT_OPTIONS = { open_timeout: 10, @@ -40,70 +33,63 @@ module Gitlab Net::HTTP::Trace ].freeze - include HTTParty # rubocop:disable Gitlab/HTTParty + # We are explicitly assigning these constants because they are used in the codebase. + Error = HTTParty::Error + Response = HTTParty::Response + ResponseError = HTTParty::ResponseError + CookieHash = HTTParty::CookieHash class << self - alias_method :httparty_perform_request, :perform_request - end - - connection_adapter ::Gitlab::HTTPConnectionAdapter - - def self.perform_request(http_method, path, options, &block) - raise_if_blocked_by_silent_mode(http_method) - - log_info = options.delete(:extra_log_info) - options_with_timeouts = - if !options.has_key?(:timeout) - options.with_defaults(DEFAULT_TIMEOUT_OPTIONS) - else - options + ::Gitlab::HTTP_V2::SUPPORTED_HTTP_METHODS.each do |method| + define_method(method) do |path, options = {}, &block| + if ::Feature.enabled?(:use_gitlab_http_v2, Feature.current_request) + ::Gitlab::HTTP_V2.public_send(method, path, http_v2_options(options), &block) # rubocop:disable GitlabSecurity/PublicSend + else + ::Gitlab::LegacyHTTP.public_send(method, path, options, &block) # rubocop:disable GitlabSecurity/PublicSend + end end + end - if options[:stream_body] - return httparty_perform_request(http_method, path, options_with_timeouts, &block) + def try_get(path, options = {}, &block) + get(path, options, &block) + rescue *HTTP_ERRORS + nil end - start_time = nil - read_total_timeout = options.fetch(:timeout, DEFAULT_READ_TOTAL_TIMEOUT) + # TODO: This method is subject to be removed + # We have this for now because we explicitly use the `perform_request` method in some places. + def perform_request(http_method, path, options, &block) + if ::Feature.enabled?(:use_gitlab_http_v2, Feature.current_request) + method_name = http_method::METHOD.downcase.to_sym - httparty_perform_request(http_method, path, options_with_timeouts) do |fragment| - start_time ||= Gitlab::Metrics::System.monotonic_time - elapsed = Gitlab::Metrics::System.monotonic_time - start_time + unless ::Gitlab::HTTP_V2::SUPPORTED_HTTP_METHODS.include?(method_name) + raise ArgumentError, "Unsupported HTTP method: '#{method_name}'." + end - if elapsed > read_total_timeout - raise ReadTotalTimeout, "Request timed out after #{elapsed} seconds" + # Use `::Gitlab::HTTP_V2.get/post/...` methods + ::Gitlab::HTTP_V2.public_send(method_name, path, http_v2_options(options), &block) # rubocop:disable GitlabSecurity/PublicSend + else + ::Gitlab::LegacyHTTP.perform_request(http_method, path, options, &block) end - - yield fragment if block end - rescue HTTParty::RedirectionTooDeep - raise RedirectionTooDeep - rescue *HTTP_ERRORS => e - extra_info = log_info || {} - extra_info = log_info.call(e, path, options) if log_info.respond_to?(:call) - Gitlab::ErrorTracking.log_exception(e, extra_info) - raise e - end - - def self.try_get(path, options = {}, &block) - self.get(path, options, &block) - rescue *HTTP_ERRORS - nil - end - def self.raise_if_blocked_by_silent_mode(http_method) - return unless blocked_by_silent_mode?(http_method) + private - ::Gitlab::SilentMode.log_info( - message: 'Outbound HTTP request blocked', - outbound_http_request_method: http_method.to_s - ) - - raise SilentModeBlockedError, 'only get, head, options, and trace methods are allowed in silent mode' - end + def http_v2_options(options) + # TODO: until we remove `allow_object_storage` from all places. + if options.delete(:allow_object_storage) + options[:extra_allowed_uris] = ObjectStoreSettings.enabled_endpoint_uris + end - def self.blocked_by_silent_mode?(http_method) - ::Gitlab::SilentMode.enabled? && SILENT_MODE_ALLOWED_METHODS.exclude?(http_method) + # Configure HTTP_V2 Client + { + allow_local_requests: Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services?, + deny_all_requests_except_allowed: Gitlab::CurrentSettings.deny_all_requests_except_allowed?, + dns_rebinding_protection_enabled: Gitlab::CurrentSettings.dns_rebinding_protection_enabled?, + outbound_local_requests_allowlist: Gitlab::CurrentSettings.outbound_local_requests_whitelist, # rubocop:disable Naming/InclusiveLanguage + silent_mode_enabled: Gitlab::SilentMode.enabled? + }.merge(options) + end end end end diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 822b8a9f8d9..8e9a63a9f7f 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -56,7 +56,7 @@ module Gitlab allow_object_storage: allow_object_storage?, dns_rebind_protection: dns_rebind_protection?, schemes: %w[http https]) - rescue Gitlab::UrlBlocker::BlockedUrlError => e + rescue Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError => e raise Gitlab::HTTP::BlockedUrlError, "URL is blocked: #{e.message}" end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index f8e7e66a8a5..96e3d90c139 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -45,7 +45,7 @@ module Gitlab 'bg' => 0, 'cs_CZ' => 0, 'da_DK' => 29, - 'de' => 96, + 'de' => 97, 'en' => 100, 'eo' => 0, 'es' => 28, @@ -54,20 +54,20 @@ module Gitlab 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 99, + 'ja' => 98, 'ko' => 23, 'nb_NO' => 21, 'nl_NL' => 0, 'pl_PL' => 3, - 'pt_BR' => 55, + 'pt_BR' => 57, 'ro_RO' => 76, - 'ru' => 22, + 'ru' => 21, 'si_LK' => 12, 'tr_TR' => 8, - 'uk' => 51, + 'uk' => 52, 'zh_CN' => 99, 'zh_HK' => 1, - 'zh_TW' => 99 + 'zh_TW' => 100 }.freeze private_constant :TRANSLATION_LEVELS @@ -81,6 +81,13 @@ module Gitlab TRANSLATION_LEVELS.fetch(code, 0) end + def trimmed_language_name(code) + language_name = AVAILABLE_LANGUAGES[code] + return if language_name.blank? + + language_name.sub(/\s-\s.*/, '') + end + def available_locales AVAILABLE_LANGUAGES.keys end diff --git a/lib/gitlab/i18n/po_linter.rb b/lib/gitlab/i18n/po_linter.rb index 3ad01ef2257..8092b3c72ea 100644 --- a/lib/gitlab/i18n/po_linter.rb +++ b/lib/gitlab/i18n/po_linter.rb @@ -9,7 +9,7 @@ module Gitlab attr_reader :po_path, :translation_entries, :metadata_entry, :locale - VARIABLE_REGEX = /%{\w*}|%[a-z]/.freeze + VARIABLE_REGEX = /%{\w*}|%[a-z]/ def initialize(po_path:, locale: I18n.locale.to_s) @po_path = po_path diff --git a/lib/gitlab/i18n/translation_entry.rb b/lib/gitlab/i18n/translation_entry.rb index 6623d42f526..6f4451d42ee 100644 --- a/lib/gitlab/i18n/translation_entry.rb +++ b/lib/gitlab/i18n/translation_entry.rb @@ -3,8 +3,8 @@ module Gitlab module I18n class TranslationEntry - PERCENT_REGEX = /(?:^|[^%])%(?!{\w*}|[a-z%])/.freeze - ANGLE_BRACKET_REGEX = /[<>]/.freeze + PERCENT_REGEX = /(?:^|[^%])%(?!{\w*}|[a-z%])/ + ANGLE_BRACKET_REGEX = /[<>]/ attr_reader :nplurals, :entry_data diff --git a/lib/gitlab/import/import_failure_service.rb b/lib/gitlab/import/import_failure_service.rb index 714d9b3edd9..a8ccf24b780 100644 --- a/lib/gitlab/import/import_failure_service.rb +++ b/lib/gitlab/import/import_failure_service.rb @@ -9,7 +9,8 @@ module Gitlab project_id: nil, error_source: nil, fail_import: false, - metrics: false + metrics: false, + external_identifiers: {} ) new( exception: exception, @@ -17,7 +18,8 @@ module Gitlab project_id: project_id, error_source: error_source, fail_import: fail_import, - metrics: metrics + metrics: metrics, + external_identifiers: external_identifiers ).execute end @@ -27,7 +29,8 @@ module Gitlab project_id: nil, error_source: nil, fail_import: false, - metrics: false + metrics: false, + external_identifiers: {} ) if import_state.blank? && project_id.blank? @@ -46,6 +49,7 @@ module Gitlab @error_source = error_source @fail_import = fail_import @metrics = metrics + @external_identifiers = external_identifiers end def execute @@ -58,19 +62,20 @@ module Gitlab private - attr_reader :exception, :import_state, :project, :error_source, :fail_import, :metrics + attr_reader :exception, :import_state, :project, :error_source, :fail_import, :metrics, :external_identifiers def track_exception attributes = { import_type: project.import_type, project_id: project.id, - source: error_source + source: error_source, + external_identifiers: external_identifiers } Gitlab::Import::Logger.error( attributes.merge( message: 'importer failed', - 'error.message': exception.message + 'exception.message': exception.message ) ) @@ -85,7 +90,8 @@ module Gitlab exception_class: exception.class.to_s, exception_message: exception.message.truncate(255), correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, - retry_count: fail_import ? 0 : nil + retry_count: fail_import ? 0 : nil, + external_identifiers: external_identifiers ) end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 7130fd8d7d6..6f3601e9a21 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -172,6 +172,7 @@ included_attributes: - :project_id - :ref - :updated_at + - :owner_id error_tracking_setting: - :api_url - :organization_name diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 78d0735bbb5..943c997a056 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -194,6 +194,7 @@ module Gitlab def setup_pipeline_schedule @relation_hash['active'] = false + @relation_hash['owner_id'] = @user.id end def setup_merge_request diff --git a/lib/gitlab/internal_events.rb b/lib/gitlab/internal_events.rb index 92bf2a826ff..2790bc8ee24 100644 --- a/lib/gitlab/internal_events.rb +++ b/lib/gitlab/internal_events.rb @@ -9,10 +9,31 @@ module Gitlab class << self include Gitlab::Tracking::Helpers - def track_event(event_name, **kwargs) + def track_event(event_name, send_snowplow_event: true, **kwargs) raise UnknownEventError, "Unknown event: #{event_name}" unless EventDefinitions.known_event?(event_name) + increase_total_counter(event_name) + update_unique_counter(event_name, kwargs) + trigger_snowplow_event(event_name, kwargs) if send_snowplow_event + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_name: event_name, kwargs: kwargs) + nil + end + + private + + def increase_total_counter(event_name) + return unless ::ServicePing::ServicePingSettings.enabled? + + redis_counter_key = + Gitlab::Usage::Metrics::Instrumentations::TotalCountMetric.redis_key(event_name) + Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } + end + + def update_unique_counter(event_name, kwargs) unique_property = EventDefinitions.unique_property(event_name) + return unless unique_property + unique_method = :id unless kwargs.has_key?(unique_property) @@ -26,7 +47,9 @@ module Gitlab unique_value = kwargs[unique_property].public_send(unique_method) # rubocop:disable GitlabSecurity/PublicSend UsageDataCounters::HLLRedisCounter.track_event(event_name, values: unique_value) + end + def trigger_snowplow_event(event_name, kwargs) user = kwargs[:user] project = kwargs[:project] namespace = kwargs[:namespace] @@ -44,13 +67,8 @@ module Gitlab ).to_context track_struct_event(event_name, contexts: [standard_context, service_ping_context]) - rescue StandardError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_name: event_name, kwargs: kwargs) - nil end - private - def track_struct_event(event_name, contexts:) category = 'InternalEventTracking' tracker = Gitlab::Tracking.tracker diff --git a/lib/gitlab/internal_events/event_definitions.rb b/lib/gitlab/internal_events/event_definitions.rb index f3c8092bcb0..9fd58ae0cb0 100644 --- a/lib/gitlab/internal_events/event_definitions.rb +++ b/lib/gitlab/internal_events/event_definitions.rb @@ -16,7 +16,7 @@ module Gitlab def unique_property(event_name) unique_value = events[event_name]&.to_s - raise(InvalidMetricConfiguration, "Unique property not defined for #{event_name}") unless unique_value + return unless unique_value unless VALID_UNIQUE_VALUES.include?(unique_value) raise(InvalidMetricConfiguration, "Invalid unique value '#{unique_value}' for #{event_name}") @@ -32,7 +32,7 @@ module Gitlab private def events - load_configurations if @events.nil? || Gitlab::Usage::MetricDefinition.metric_definitions_changed? + load_configurations if @events.nil? @events end diff --git a/lib/gitlab/jira/dvcs.rb b/lib/gitlab/jira/dvcs.rb index 41a039674b3..020c31fa281 100644 --- a/lib/gitlab/jira/dvcs.rb +++ b/lib/gitlab/jira/dvcs.rb @@ -5,7 +5,7 @@ module Gitlab module Dvcs ENCODED_SLASH = '@' SLASH = '/' - ENCODED_ROUTE_REGEX = /[a-zA-Z0-9_\-\.#{ENCODED_SLASH}]+/.freeze + ENCODED_ROUTE_REGEX = /[a-zA-Z0-9_\-\.#{ENCODED_SLASH}]+/ def self.encode_slash(path) path.gsub(SLASH, ENCODED_SLASH) diff --git a/lib/gitlab/legacy_http.rb b/lib/gitlab/legacy_http.rb new file mode 100644 index 00000000000..f38b2819c15 --- /dev/null +++ b/lib/gitlab/legacy_http.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +# +# IMPORTANT: With the new development of the 'gitlab-http' gem (https://gitlab.com/gitlab-org/gitlab/-/issues/415686), +# no additional change should be implemented in this class. This class will be removed after migrating all +# the usages to the new gem. +# + +require_relative 'http_connection_adapter' + +module Gitlab + class LegacyHTTP # rubocop:disable Gitlab/NamespacedClass + include HTTParty # rubocop:disable Gitlab/HTTParty + + class << self + alias_method :httparty_perform_request, :perform_request + end + + connection_adapter ::Gitlab::HTTPConnectionAdapter + + def self.perform_request(http_method, path, options, &block) + raise_if_blocked_by_silent_mode(http_method) + + log_info = options.delete(:extra_log_info) + options_with_timeouts = + if !options.has_key?(:timeout) + options.with_defaults(Gitlab::HTTP::DEFAULT_TIMEOUT_OPTIONS) + else + options + end + + return httparty_perform_request(http_method, path, options_with_timeouts, &block) if options[:stream_body] + + start_time = nil + read_total_timeout = options.fetch(:timeout, Gitlab::HTTP::DEFAULT_READ_TOTAL_TIMEOUT) + + httparty_perform_request(http_method, path, options_with_timeouts) do |fragment| + start_time ||= Gitlab::Metrics::System.monotonic_time + elapsed = Gitlab::Metrics::System.monotonic_time - start_time + + if elapsed > read_total_timeout + raise Gitlab::HTTP::ReadTotalTimeout, "Request timed out after #{elapsed} seconds" + end + + yield fragment if block + end + rescue HTTParty::RedirectionTooDeep + raise Gitlab::HTTP::RedirectionTooDeep + rescue *Gitlab::HTTP::HTTP_ERRORS => e + extra_info = log_info || {} + extra_info = log_info.call(e, path, options) if log_info.respond_to?(:call) + Gitlab::ErrorTracking.log_exception(e, extra_info) + raise e + end + + def self.try_get(path, options = {}, &block) + self.get(path, options, &block) # rubocop:disable Style/RedundantSelf + rescue *Gitlab::HTTP::HTTP_ERRORS + nil + end + + def self.raise_if_blocked_by_silent_mode(http_method) + return unless blocked_by_silent_mode?(http_method) + + ::Gitlab::SilentMode.log_info( + message: 'Outbound HTTP request blocked', + outbound_http_request_method: http_method.to_s + ) + + raise Gitlab::HTTP::SilentModeBlockedError, + 'only get, head, options, and trace methods are allowed in silent mode' + end + + def self.blocked_by_silent_mode?(http_method) + ::Gitlab::SilentMode.enabled? && Gitlab::HTTP::SILENT_MODE_ALLOWED_METHODS.exclude?(http_method) + end + end +end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 5f760e764c8..426d449abbe 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -11,20 +11,6 @@ require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) # This service is run independently of the main Rails process, # therefore the `Rails` class and its methods are unavailable. -# TODO: Remove this once we're on Ruby 3 -# https://gitlab.com/gitlab-org/gitlab/-/issues/393651 -unless YAML.respond_to?(:safe_load_file) - module YAML - # Temporary Ruby 2 back-compat workaround. - # - # This method only exists as of stdlib 3.0.0: - # https://ruby-doc.org/stdlib-3.0.0/libdoc/psych/rdoc/Psych.html - def self.safe_load_file(path, **options) - YAML.safe_load(File.read(path), **options) - end - end -end - module Gitlab module MailRoom RAILS_ROOT_DIR = Pathname.new('../..').expand_path(__dir__).freeze diff --git a/lib/gitlab/merge_requests/mergeability/check_result.rb b/lib/gitlab/merge_requests/mergeability/check_result.rb index fae4b721e1a..e18909d8f17 100644 --- a/lib/gitlab/merge_requests/mergeability/check_result.rb +++ b/lib/gitlab/merge_requests/mergeability/check_result.rb @@ -35,6 +35,10 @@ module Gitlab { status: status, payload: payload } end + def identifier + payload&.fetch(:identifier)&.to_sym + end + def failed? status == FAILED_STATUS end diff --git a/lib/gitlab/merge_requests/message_generator.rb b/lib/gitlab/merge_requests/message_generator.rb index 5ca26fdae86..2307d4fc64e 100644 --- a/lib/gitlab/merge_requests/message_generator.rb +++ b/lib/gitlab/merge_requests/message_generator.rb @@ -111,7 +111,7 @@ module Gitlab all_commits ].freeze - PLACEHOLDERS_COMBINED_REGEX = /%{(#{Regexp.union(PLACEHOLDERS.keys)})}/.freeze + PLACEHOLDERS_COMBINED_REGEX = /%{(#{Regexp.union(PLACEHOLDERS.keys)})}/ def replace_placeholders(message, allowed_placeholders: [], squash: false, keep_carriage_return: false) # Convert CRLF to LF. diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index 858a0a120cc..e22b9c2a761 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -7,7 +7,7 @@ module Gitlab module Metrics module Exporter class BaseExporter < Daemon - CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze + CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/ attr_reader :server diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index c299fa37e7a..b00459de17e 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -13,7 +13,7 @@ module Gitlab "put" => %w[200 202 204 400 401 403 404 405 406 409 410 422 500] }.freeze - HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$}.freeze + HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$} FEATURE_CATEGORY_DEFAULT = ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT ENDPOINT_MISSING = 'unknown' diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index d818aa43853..c806db09e66 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -40,7 +40,7 @@ module Gitlab private def puma_stats - Puma.stats + ::Puma.stats rescue NoMethodError Gitlab::AppLogger.info "PumaSampler: stats are not available yet, waiting for Puma to boot" nil diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index f9749b65888..c7c54efc50e 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -10,7 +10,7 @@ module Gitlab attach_to :active_record DB_COUNTERS = %i[count write_count cached_count].freeze - SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i.freeze + SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze @@ -19,7 +19,7 @@ module Gitlab DB_LOAD_BALANCING_COUNTERS = %i[count cached_count wal_count wal_cached_count].freeze DB_LOAD_BALANCING_DURATIONS = %i[duration_s].freeze - SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze + SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/ # This event is published from ActiveRecordBaseTransactionMetrics and # used to record a database transaction duration when calling diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 9b0ae84dec2..80ce155321b 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -16,13 +16,13 @@ module Gitlab PROC_FD_GLOB = '/proc/self/fd/*' PROC_MEM_INFO = '/proc/meminfo' - PRIVATE_PAGES_PATTERN = /^(Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/.freeze - PSS_PATTERN = /^Pss:\s+(?<value>\d+)/.freeze - RSS_TOTAL_PATTERN = /^VmRSS:\s+(?<value>\d+)/.freeze - RSS_ANON_PATTERN = /^RssAnon:\s+(?<value>\d+)/.freeze - RSS_FILE_PATTERN = /^RssFile:\s+(?<value>\d+)/.freeze - MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/.freeze - MEM_TOTAL_PATTERN = /^MemTotal:\s+(?<value>\d+) (.+)/.freeze + PRIVATE_PAGES_PATTERN = /^(Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/ + PSS_PATTERN = /^Pss:\s+(?<value>\d+)/ + RSS_TOTAL_PATTERN = /^VmRSS:\s+(?<value>\d+)/ + RSS_ANON_PATTERN = /^RssAnon:\s+(?<value>\d+)/ + RSS_FILE_PATTERN = /^RssFile:\s+(?<value>\d+)/ + MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/ + MEM_TOTAL_PATTERN = /^MemTotal:\s+(?<value>\d+) (.+)/ def summary proportional_mem = memory_usage_uss_pss diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index f3c1e6897af..ad635a9e376 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -9,7 +9,7 @@ module Gitlab # etc. class WebTransaction < Transaction THREAD_KEY = :_gitlab_metrics_transaction - BASE_LABEL_KEYS = %i[controller action feature_category].freeze + BASE_LABEL_KEYS = %i[controller action feature_category endpoint_id].freeze CONTROLLER_KEY = 'action_controller.instance' ENDPOINT_KEY = 'api.endpoint' @@ -95,7 +95,13 @@ module Gitlab action = "#{action}.#{suffix}" end - { controller: controller.class.name, action: action, feature_category: feature_category } + { + controller: controller.class.name, + action: action, + feature_category: feature_category, + # inline endpoint_id_for_action as not all controllers extend ApplicationController + endpoint_id: "#{controller.class.name}##{controller.action_name}" + } end def labels_from_endpoint @@ -112,7 +118,12 @@ module Gitlab if route path = endpoint_paths_cache[route.request_method][route.path] - { controller: 'Grape', action: "#{route.request_method} #{path}", feature_category: feature_category } + { + controller: 'Grape', + action: "#{route.request_method} #{path}", + feature_category: feature_category, + endpoint_id: API::Base.endpoint_id_for_route(route) + } end end diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb index 1f15f1d5857..1131e34b73c 100644 --- a/lib/gitlab/middleware/compressed_json.rb +++ b/lib/gitlab/middleware/compressed_json.rb @@ -6,23 +6,23 @@ module Gitlab INSTANCE_PACKAGES_PATH = %r{ \A/api/v4/packages/npm/-/npm/v1/security/ (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) - }xi.freeze + }xi GROUP_PACKAGES_PATH = %r{ \A/api/v4/groups/ (?<id> [a-zA-Z0-9%-._]{1,255} )/-/packages/npm/-/npm/v1/security/ (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) - }xi.freeze + }xi PROJECT_PACKAGES_PATH = %r{ \A/api/v4/projects/ (?<id> [a-zA-Z0-9%-._]{1,255} )/packages/npm/-/npm/v1/security/ (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) - }xi.freeze + }xi MAXIMUM_BODY_SIZE = 200.kilobytes.to_i - UNSAFE_CHARACTERS = %r{[!"#&'()*+,./:;<>=?@\[\]^`{}|~$]}xi.freeze + UNSAFE_CHARACTERS = %r{[!"#&'()*+,./:;<>=?@\[\]^`{}|~$]}xi def initialize(app) @app = app diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 4da5fef9fd7..d2336ec4bb2 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -8,7 +8,7 @@ module Gitlab include ActionView::Helpers::TagHelper include ActionController::HttpAuthentication::Basic - PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/}.freeze + PROJECT_PATH_REGEX = %r{\A(#{Gitlab::PathRegex.full_namespace_route_regex}/#{Gitlab::PathRegex.project_route_regex})/} def initialize(app) @app = app diff --git a/lib/gitlab/middleware/handle_malformed_strings.rb b/lib/gitlab/middleware/handle_malformed_strings.rb index b966395ee32..e0f38b63cc1 100644 --- a/lib/gitlab/middleware/handle_malformed_strings.rb +++ b/lib/gitlab/middleware/handle_malformed_strings.rb @@ -37,7 +37,7 @@ module Gitlab request.params.values.any? do |value| param_has_null_byte?(value) end - rescue ActionController::BadRequest + rescue ActionController::BadRequest, ActionDispatch::Http::Parameters::ParseError # If we can't build an ActionDispatch::Request something's wrong # This would also happen if `#params` contains invalid UTF-8 # in this case we'll return a 400 diff --git a/lib/gitlab/middleware/path_traversal_check.rb b/lib/gitlab/middleware/path_traversal_check.rb new file mode 100644 index 00000000000..79465f3cb30 --- /dev/null +++ b/lib/gitlab/middleware/path_traversal_check.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + class PathTraversalCheck + PATH_TRAVERSAL_MESSAGE = 'Potential path traversal attempt detected' + + def initialize(app) + @app = app + end + + def call(env) + if Feature.enabled?(:check_path_traversal_middleware, Feature.current_request) + log_params = {} + + execution_time = measure_execution_time do + check(env, log_params) + end + + log_params[:duration_ms] = execution_time.round(5) if execution_time + + log(log_params) unless log_params.empty? + end + + @app.call(env) + end + + private + + def measure_execution_time(&blk) + if Feature.enabled?(:log_execution_time_path_traversal_middleware, Feature.current_request) + Benchmark.ms(&blk) + else + yield + + nil + end + end + + def check(env, log_params) + request = ::Rack::Request.new(env) + fullpath = request.fullpath + decoded_fullpath = CGI.unescape(fullpath) + ::Gitlab::PathTraversal.check_path_traversal!(decoded_fullpath, skip_decoding: true) + + rescue ::Gitlab::PathTraversal::PathTraversalAttackError + log_params[:fullpath] = fullpath + log_params[:message] = PATH_TRAVERSAL_MESSAGE + end + + def log(payload) + Gitlab::AppLogger.warn( + payload.merge(class_name: self.class.name) + ) + end + end + end +end diff --git a/lib/gitlab/middleware/read_only.rb b/lib/gitlab/middleware/read_only.rb index 8e17073abab..83c52a6c6e0 100644 --- a/lib/gitlab/middleware/read_only.rb +++ b/lib/gitlab/middleware/read_only.rb @@ -3,7 +3,7 @@ module Gitlab module Middleware class ReadOnly - API_VERSIONS = (3..4).freeze + API_VERSIONS = (3..4) def self.internal_routes @internal_routes ||= diff --git a/lib/gitlab/middleware/sidekiq_web_static.rb b/lib/gitlab/middleware/sidekiq_web_static.rb index c5d2ecbe00e..7640c02fbb5 100644 --- a/lib/gitlab/middleware/sidekiq_web_static.rb +++ b/lib/gitlab/middleware/sidekiq_web_static.rb @@ -8,7 +8,7 @@ module Gitlab module Middleware class SidekiqWebStatic - SIDEKIQ_REGEX = %r{\A/admin/sidekiq/}.freeze + SIDEKIQ_REGEX = %r{\A/admin/sidekiq/} def initialize(app) @app = app diff --git a/lib/gitlab/middleware/static.rb b/lib/gitlab/middleware/static.rb index 324d929a93d..e6e36de175d 100644 --- a/lib/gitlab/middleware/static.rb +++ b/lib/gitlab/middleware/static.rb @@ -3,7 +3,7 @@ module Gitlab module Middleware class Static < ActionDispatch::Static - UPLOADS_REGEX = %r{\A/uploads(/|\z)}.freeze + UPLOADS_REGEX = %r{\A/uploads(/|\z)} def call(env) return @app.call(env) if UPLOADS_REGEX.match?(env['PATH_INFO']) diff --git a/lib/gitlab/observability.rb b/lib/gitlab/observability.rb index a4e18cc170b..d42d10cd0f4 100644 --- a/lib/gitlab/observability.rb +++ b/lib/gitlab/observability.rb @@ -4,15 +4,6 @@ module Gitlab module Observability extend self - ACTION_TO_PERMISSION = { - explore: :read_observability, - datasources: :admin_observability, - manage: :admin_observability, - dashboards: :read_observability - }.freeze - - EMBEDDABLE_PATHS = %w[explore goto].freeze - # Returns the GitLab Observability URL # def observability_url @@ -31,124 +22,13 @@ module Gitlab "#{Gitlab::Observability.observability_url}/v3/tenant/#{project.id}" end - # Returns true if the GitLab Observability UI (GOUI) feature flag is enabled - # - # @deprecated - # - def enabled?(group = nil) - return Feature.enabled?(:observability_group_tab, group) if group - - Feature.enabled?(:observability_group_tab) - end - - # Returns the embeddable Observability URL of a given URL - # - # - Validates the URL - # - Checks that the path is embeddable - # - Converts the gitlab.com URL to observe.gitlab.com URL - # - # e.g. - # - # from: gitlab.com/groups/GROUP_PATH/-/observability/explore?observability_path=/explore - # to observe.gitlab.com/-/GROUP_ID/explore - # - # Returns nil if the URL is not a valid Observability URL or the path is not embeddable - # - def embeddable_url(url) - uri = validate_url(url, Gitlab.config.gitlab.url) - return unless uri - - group = group_from_observability_url(url) - return unless group - - parsed_query = CGI.parse(uri.query.to_s).transform_values(&:first).symbolize_keys - observability_path = parsed_query[:observability_path] - - return build_full_url(group, observability_path, '/') if observability_path_embeddable?(observability_path) - end - - # Returns true if the user is allowed to perform an action within a group - # - def allowed_for_action?(user, group, action) - return false if action.nil? - - permission = ACTION_TO_PERMISSION.fetch(action.to_sym, :admin_observability) - allowed?(user, group, permission) - end - - # Returns true if the user has the specified permission within the group - def allowed?(user, group, permission = :admin_observability) - return false unless group && user - - observability_url.present? && Ability.allowed?(user, permission, group) - end - - # Builds the full Observability URL given a certan group and path - # - # If unsanitized_observability_path is not valid or missing, fallbacks to fallback_path - # - def build_full_url(group, unsanitized_observability_path, fallback_path) - return unless group - - # When running Observability UI in standalone mode (i.e. not backed by Observability Backend) - # the group-id is not required. !!This is only used for local dev!! - base_url = ENV['STANDALONE_OBSERVABILITY_UI'] == 'true' ? observability_url : "#{observability_url}/-/#{group.id}" - - sanitized_path = if unsanitized_observability_path && sanitize(unsanitized_observability_path) != '' - CGI.unescapeHTML(sanitize(unsanitized_observability_path)) - else - fallback_path || '/' - end - - sanitized_path.prepend('/') if sanitized_path[0] != '/' - - "#{base_url}#{sanitized_path}" - end - - private - - def validate_url(url, reference_url) - uri = URI.parse(url) - reference_uri = URI.parse(reference_url) - - return uri if uri.scheme == reference_uri.scheme && - uri.port == reference_uri.port && - uri.host.casecmp?(reference_uri.host) - rescue URI::InvalidURIError - nil - end - - def link_sanitizer - @link_sanitizer ||= Rails::Html::Sanitizer.safe_list_sanitizer.new - end - - def sanitize(input) - link_sanitizer.sanitize(input, {})&.html_safe - end - - def group_from_observability_url(url) - match = Rails.application.routes.recognize_path(url) - - return if match[:unmatched_route].present? - return if match[:group_id].blank? || match[:action].blank? || match[:controller] != "groups/observability" - - group_path = match[:group_id] - Group.find_by_full_path(group_path) - rescue ActionController::RoutingError - nil - end - - def observability_path_embeddable?(observability_path) - return false unless observability_path - - observability_path = observability_path[1..] if observability_path[0] == '/' - - parsed_observability_path = URI.parse(observability_path).path.split('/') - - base_path = parsed_observability_path[0] + def should_enable_observability_auth_scopes?(resource) + # Enable the needed oauth scopes if tracing is enabled. + if resource.is_a?(Group) || resource.is_a?(Project) + return Feature.enabled?(:observability_tracing, + resource.root_ancestor) + end - EMBEDDABLE_PATHS.include?(base_path) - rescue URI::InvalidURIError false end end diff --git a/lib/gitlab/pages/cache_control.rb b/lib/gitlab/pages/cache_control.rb deleted file mode 100644 index 81da34f1219..00000000000 --- a/lib/gitlab/pages/cache_control.rb +++ /dev/null @@ -1,105 +0,0 @@ -# frozen_string_literal: true - -require 'set' - -module Gitlab - module Pages - class CacheControl - include Gitlab::Utils::StrongMemoize - - EXPIRE = 12.hours - # To avoid delivering expired deployment URL in the cached payload, - # use a longer expiration time in the deployment URL - DEPLOYMENT_EXPIRATION = (EXPIRE + 12.hours) - - SETTINGS_CACHE_KEY = 'pages_domain_for_%{type}_%{id}' - PAYLOAD_CACHE_KEY = '%{settings_cache_key}_%{settings_hash}' - - class << self - def for_domain(domain_id) - new(type: :domain, id: domain_id) - end - - def for_namespace(namespace_id) - new(type: :namespace, id: namespace_id) - end - end - - def initialize(type:, id:) - raise(ArgumentError, "type must be :namespace or :domain") unless %i[namespace domain].include?(type) - - @type = type - @id = id - end - - def cache_key - strong_memoize(:payload_cache_key) do - cache_settings_hash! - - payload_cache_key_for(settings_hash) - end - end - - # Invalidates the cache. - # - # Since rails nodes and sidekiq nodes have different application settings, - # and the invalidation happens in a sidekiq node, we have to use the - # cached settings hash to build the payload cache key to be invalidated. - def clear_cache - keys = cached_settings_hashes - .map { |hash| payload_cache_key_for(hash) } - .push(settings_cache_key) - - ::Gitlab::AppLogger.info( - message: 'clear pages cache', - pages_keys: keys, - pages_type: @type, - pages_id: @id - ) - - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - Rails.cache.delete_multi(keys) - end - end - - private - - # Since rails nodes and sidekiq nodes have different application settings, - # we cache the application settings hash when creating the payload cache - # so we can use these values to invalidate the cache in a sidekiq node later. - def cache_settings_hash! - cached = cached_settings_hashes.to_set - Rails.cache.write(settings_cache_key, cached.add(settings_hash)) - end - - def cached_settings_hashes - Rails.cache.read(settings_cache_key) || [] - end - - def payload_cache_key_for(settings_hash) - PAYLOAD_CACHE_KEY % { - settings_cache_key: settings_cache_key, - settings_hash: settings_hash - } - end - - def settings_cache_key - strong_memoize(:settings_cache_key) do - SETTINGS_CACHE_KEY % { type: @type, id: @id } - end - end - - def settings_hash - strong_memoize(:settings_hash) do - values = ::Gitlab.config.pages.dup - - values['app_settings'] = ::Gitlab::CurrentSettings.attributes.slice( - 'force_pages_access_control' - ) - - ::Digest::SHA256.hexdigest(values.inspect) - end - end - end - end -end diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index 592f635c14e..81dcc54ff35 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -3,13 +3,6 @@ module Gitlab module Pagination module CursorBasedKeyset - SUPPORTED_ORDERING = { - Group => { name: :asc }, - AuditEvent => { id: :desc }, - ::Ci::Build => { id: :desc }, - ::Packages::BuildInfo => { id: :desc } - }.freeze - SUPPORTED_MULTI_ORDERING = { Group => { name: [:asc] }, AuditEvent => { id: [:desc] }, @@ -33,11 +26,7 @@ module Gitlab ENFORCED_TYPES = [Group].freeze def self.available_for_type?(relation) - if Feature.enabled?(:api_keyset_pagination_multi_order) - SUPPORTED_MULTI_ORDERING.key?(relation.klass) - else - SUPPORTED_ORDERING.key?(relation.klass) - end + SUPPORTED_MULTI_ORDERING.key?(relation.klass) end def self.available?(cursor_based_request_context, relation) @@ -50,16 +39,10 @@ module Gitlab end def self.order_satisfied?(relation, cursor_based_request_context) - if Feature.enabled?(:api_keyset_pagination_multi_order) - order_by_from_request = cursor_based_request_context.order - sort_from_request = cursor_based_request_context.sort - - SUPPORTED_MULTI_ORDERING[relation.klass][order_by_from_request]&.include?(sort_from_request) - else - order_by_from_request = cursor_based_request_context.order_by + order_by_from_request = cursor_based_request_context.order + sort_from_request = cursor_based_request_context.sort - SUPPORTED_ORDERING[relation.klass] == order_by_from_request - end + SUPPORTED_MULTI_ORDERING[relation.klass][order_by_from_request]&.include?(sort_from_request) end private_class_method :order_satisfied? end diff --git a/lib/gitlab/patch/hangouts_chat_http_override.rb b/lib/gitlab/patch/hangouts_chat_http_override.rb deleted file mode 100644 index 20dc678e251..00000000000 --- a/lib/gitlab/patch/hangouts_chat_http_override.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Patch - module HangoutsChatHTTPOverride - attr_reader :uri - - # See https://github.com/enzinia/hangouts-chat/blob/6a509f61a56e757f8f417578b393b94423831ff7/lib/hangouts_chat/http.rb - def post(payload) - httparty_response = Gitlab::HTTP.post( - uri, - body: payload.to_json, - headers: { 'Content-Type' => 'application/json' }, - parse: nil # Disables automatic response parsing - ) - httparty_response.response - # The rest of the integration expects a Net::HTTP response - end - end - end -end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 8afcf682d5d..cde621bc9e4 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -134,10 +134,10 @@ module Gitlab PATH_REGEX_STR = PATH_START_CHAR + '[a-zA-Z0-9_\-\.]' + "{0,#{Namespace::URL_MAX_LENGTH - 1}}" NAMESPACE_FORMAT_REGEX_JS = PATH_REGEX_STR + '[a-zA-Z0-9_\-]|[a-zA-Z0-9_]' - NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/.freeze - NAMESPACE_FORMAT_REGEX = /(?:#{NAMESPACE_FORMAT_REGEX_JS})#{NO_SUFFIX_REGEX}/.freeze - PROJECT_PATH_FORMAT_REGEX = /(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX}/.freeze - FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/){,#{Namespace::NUMBER_OF_ANCESTORS_ALLOWED}}#{NAMESPACE_FORMAT_REGEX}}.freeze + NO_SUFFIX_REGEX = /(?<!\.git|\.atom)/ + NAMESPACE_FORMAT_REGEX = /(?:#{NAMESPACE_FORMAT_REGEX_JS})#{NO_SUFFIX_REGEX}/ + PROJECT_PATH_FORMAT_REGEX = /(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX}/ + FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/){,#{Namespace::NUMBER_OF_ANCESTORS_ALLOWED}}#{NAMESPACE_FORMAT_REGEX}} def organization_route_regex @organization_route_regex ||= begin @@ -188,19 +188,19 @@ module Gitlab end def repository_route_regex - @repository_route_regex ||= /(#{full_namespace_route_regex}|#{personal_snippet_repository_path_regex})\.*/.freeze + @repository_route_regex ||= /(#{full_namespace_route_regex}|#{personal_snippet_repository_path_regex})\.*/ end def repository_git_route_regex - @repository_git_route_regex ||= /#{repository_route_regex}\.git/.freeze + @repository_git_route_regex ||= /#{repository_route_regex}\.git/ end def repository_git_lfs_route_regex - @repository_git_lfs_route_regex ||= %r{#{repository_git_route_regex}\/(info\/lfs|gitlab-lfs)\/}.freeze + @repository_git_lfs_route_regex ||= %r{#{repository_git_route_regex}\/(info\/lfs|gitlab-lfs)\/} end def repository_wiki_git_route_regex - @repository_wiki_git_route_regex ||= /#{full_namespace_route_regex}\.*\.wiki\.git/.freeze + @repository_wiki_git_route_regex ||= /#{full_namespace_route_regex}\.*\.wiki\.git/ end def full_namespace_path_regex @@ -220,7 +220,7 @@ module Gitlab end def namespace_format_regex - @namespace_format_regex ||= /\A#{NAMESPACE_FORMAT_REGEX}\z/o.freeze + @namespace_format_regex ||= /\A#{NAMESPACE_FORMAT_REGEX}\z/o end def namespace_format_message @@ -229,7 +229,7 @@ module Gitlab end def project_path_format_regex - @project_path_format_regex ||= /\A#{PROJECT_PATH_FORMAT_REGEX}\z/o.freeze + @project_path_format_regex ||= /\A#{PROJECT_PATH_FORMAT_REGEX}\z/o end def project_path_format_message @@ -239,7 +239,7 @@ module Gitlab def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | - @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze + @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/ end def git_reference_regex @@ -270,11 +270,11 @@ module Gitlab end def container_image_regex - @container_image_regex ||= %r{([\w\.-]+\/){0,4}[\w\.-]+}.freeze + @container_image_regex ||= %r{([\w\.-]+\/){0,4}[\w\.-]+} end def container_image_blob_sha_regex - @container_image_blob_sha_regex ||= %r{[\w+.-]+:?\w+}.freeze + @container_image_blob_sha_regex ||= %r{[\w+.-]+:?\w+} end def dependency_proxy_route_regex diff --git a/lib/gitlab/path_traversal.rb b/lib/gitlab/path_traversal.rb index d42b5fde615..c8308c9da1c 100644 --- a/lib/gitlab/path_traversal.rb +++ b/lib/gitlab/path_traversal.rb @@ -15,13 +15,13 @@ module Gitlab # We url decode the path to avoid passing invalid paths forward in url encoded format. # Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580 # It also checks for backslash '\', which is sometimes a File::ALT_SEPARATOR. - def check_path_traversal!(path) + def check_path_traversal!(path, skip_decoding: false) return unless path path = path.to_s if path.is_a?(Gitlab::HashedPath) raise PathTraversalAttackError, 'Invalid path' unless path.is_a?(String) - path = ::Gitlab::Utils.decode_path(path) + path = ::Gitlab::Utils.decode_path(path) unless skip_decoding if path.match?(PATH_TRAVERSAL_REGEX) logger.warn(message: "Potential path traversal attempt detected", path: path.to_s) diff --git a/lib/gitlab/prometheus/metric_group.rb b/lib/gitlab/prometheus/metric_group.rb deleted file mode 100644 index 020d4cf74a3..00000000000 --- a/lib/gitlab/prometheus/metric_group.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - class MetricGroup - include ActiveModel::Model - - attr_accessor :name, :priority, :metrics - - validates :name, :priority, :metrics, presence: true - - def self.common_metrics - all_groups = ::PrometheusMetricsFinder.new(common: true).execute - .group_by(&:group_title) - .map do |name, metrics| - MetricGroup.new( - name: name, - priority: metrics.map(&:priority).max, - metrics: metrics.map(&:to_query_metric) - ) - end - - all_groups.sort_by(&:priority).reverse - end - - # EE only - def self.for_project(_) - common_metrics - end - end - end -end - -Gitlab::Prometheus::MetricGroup.prepend_mod_with('Gitlab::Prometheus::MetricGroup') diff --git a/lib/gitlab/prometheus/parsing_error.rb b/lib/gitlab/prometheus/parsing_error.rb deleted file mode 100644 index 20b5ef5ce55..00000000000 --- a/lib/gitlab/prometheus/parsing_error.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - ParsingError = Class.new(StandardError) - end -end diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb deleted file mode 100644 index eabac6128b5..00000000000 --- a/lib/gitlab/prometheus/queries/base_query.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - module Queries - class BaseQuery - attr_accessor :client - - delegate :query_range, :query, :label_values, :series, to: :client, prefix: true - - def raw_memory_usage_query(environment_slug) - %{avg(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}) / 2^20} - end - - def raw_cpu_usage_query(environment_slug) - %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[2m])) * 100} - end - - def initialize(client) - @client = client - end - - def query(*args) - raise NotImplementedError - end - - def self.transform_reactive_result(result) - result - end - end - end - end -end diff --git a/lib/gitlab/prometheus/queries/deployment_query.rb b/lib/gitlab/prometheus/queries/deployment_query.rb deleted file mode 100644 index 13d85d33cc0..00000000000 --- a/lib/gitlab/prometheus/queries/deployment_query.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - module Queries - class DeploymentQuery < BaseQuery - # rubocop: disable CodeReuse/ActiveRecord - def query(deployment_id) - Deployment.find_by(id: deployment_id).try do |deployment| - environment_slug = deployment.environment.slug - - memory_query = raw_memory_usage_query(environment_slug) - memory_avg_query = %{avg(avg_over_time(container_memory_usage_bytes{container_name!="POD",environment="#{environment_slug}"}[30m]))} - cpu_query = raw_cpu_usage_query(environment_slug) - cpu_avg_query = %{avg(rate(container_cpu_usage_seconds_total{container_name!="POD",environment="#{environment_slug}"}[30m])) * 100} - - timeframe_start = (deployment.created_at - 30.minutes).to_f - timeframe_end = (deployment.created_at + 30.minutes).to_f - - { - memory_values: client_query_range(memory_query, start_time: timeframe_start, end_time: timeframe_end), - memory_before: client_query(memory_avg_query, time: deployment.created_at.to_f), - memory_after: client_query(memory_avg_query, time: timeframe_end), - - cpu_values: client_query_range(cpu_query, start_time: timeframe_start, end_time: timeframe_end), - cpu_before: client_query(cpu_avg_query, time: deployment.created_at.to_f), - cpu_after: client_query(cpu_avg_query, time: timeframe_end) - } - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def self.transform_reactive_result(result) - result[:metrics] = result.delete :data - result - end - end - end - end -end diff --git a/lib/gitlab/prometheus/queries/environment_query.rb b/lib/gitlab/prometheus/queries/environment_query.rb deleted file mode 100644 index 5f3093eecd4..00000000000 --- a/lib/gitlab/prometheus/queries/environment_query.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - module Queries - class EnvironmentQuery < BaseQuery - # rubocop: disable CodeReuse/ActiveRecord - def query(environment_id) - ::Environment.find_by(id: environment_id).try do |environment| - environment_slug = environment.slug - timeframe_start = 8.hours.ago.to_f - timeframe_end = Time.now.to_f - - memory_query = raw_memory_usage_query(environment_slug) - cpu_query = raw_cpu_usage_query(environment_slug) - - { - memory_values: client_query_range(memory_query, start_time: timeframe_start, end_time: timeframe_end), - memory_current: client_query(memory_query, time: timeframe_end), - cpu_values: client_query_range(cpu_query, start_time: timeframe_start, end_time: timeframe_end), - cpu_current: client_query(cpu_query, time: timeframe_end) - } - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def self.transform_reactive_result(result) - result[:metrics] = result.delete :data - result - end - end - end - end -end diff --git a/lib/gitlab/prometheus/queries/matched_metric_query.rb b/lib/gitlab/prometheus/queries/matched_metric_query.rb deleted file mode 100644 index 73de5a11998..00000000000 --- a/lib/gitlab/prometheus/queries/matched_metric_query.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - module Queries - class MatchedMetricQuery < BaseQuery - MAX_QUERY_ITEMS = 40 - - def query - groups_data.map do |group, data| - { - group: group.name, - priority: group.priority, - active_metrics: data[:active_metrics], - metrics_missing_requirements: data[:metrics_missing_requirements] - } - end - end - - private - - def groups_data - metrics_groups = groups_with_active_metrics(Gitlab::Prometheus::MetricGroup.common_metrics) - lookup = active_series_lookup(metrics_groups) - - groups = {} - - metrics_groups.each do |group| - groups[group] ||= { active_metrics: 0, metrics_missing_requirements: 0 } - active_metrics = group.metrics.count { |metric| metric.required_metrics.all?(&lookup.method(:has_key?)) } - - groups[group][:active_metrics] += active_metrics - groups[group][:metrics_missing_requirements] += group.metrics.count - active_metrics - end - - groups - end - - def active_series_lookup(metric_groups) - timeframe_start = 8.hours.ago - timeframe_end = Time.now - - series = metric_groups.flat_map(&:metrics).flat_map(&:required_metrics).uniq - - lookup = series.each_slice(MAX_QUERY_ITEMS).flat_map do |batched_series| - client_series(*batched_series, start_time: timeframe_start, end_time: timeframe_end) - .select(&method(:has_matching_label?)) - .map { |series_info| [series_info['__name__'], true] } - end - lookup.to_h - end - - def has_matching_label?(series_info) - series_info.key?('environment') - end - - def available_metrics - @available_metrics ||= client_label_values || [] - end - - def filter_active_metrics(metric_group) - metric_group.metrics.select! do |metric| - metric.required_metrics.all?(&available_metrics.method(:include?)) - end - metric_group - end - - def groups_with_active_metrics(metric_groups) - metric_groups.map(&method(:filter_active_metrics)).select { |group| group.metrics.any? } - end - - def metrics_with_required_series(metric_groups) - metric_groups.flat_map do |group| - group.metrics.select do |metric| - metric.required_metrics.all?(&available_metrics.method(:include?)) - end - end - end - end - end - end -end diff --git a/lib/gitlab/prometheus/queries/query_additional_metrics.rb b/lib/gitlab/prometheus/queries/query_additional_metrics.rb deleted file mode 100644 index a870bb6bc5f..00000000000 --- a/lib/gitlab/prometheus/queries/query_additional_metrics.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - module Queries - module QueryAdditionalMetrics - def query_metrics(project, environment, query_context) - matched_metrics(project).map(&query_group(query_context)) - .select(&method(:group_with_any_metrics)) - end - - protected - - def query_group(query_context) - query_processor = method(:process_query).curry[query_context] - - lambda do |group| - metrics = group.metrics.map do |metric| - metric_hsh = { - title: metric.title, - weight: metric.weight, - y_label: metric.y_label, - queries: metric.queries.map(&query_processor).select(&method(:query_with_result)) - } - - metric_hsh[:id] = metric.id if metric.id - - metric_hsh - end - - { - group: group.name, - priority: group.priority, - metrics: metrics.select(&method(:metric_with_any_queries)) - } - end - end - - private - - def metric_with_any_queries(metric) - metric[:queries]&.count&.> 0 - end - - def group_with_any_metrics(group) - group[:metrics]&.count&.> 0 - end - - def query_with_result(query) - query[:result]&.any? do |item| - item&.[](:values)&.any? || item&.[](:value)&.any? - end - end - - def process_query(context, query) - query = query.dup - result = - if query.key?(:query_range) - query[:query_range] %= context - client_query_range(query[:query_range], start_time: context[:timeframe_start], end_time: context[:timeframe_end]) - else - query[:query] %= context - client_query(query[:query], time: context[:timeframe_end]) - end - - query[:result] = result&.map(&:deep_symbolize_keys) - query - end - - def available_metrics - @available_metrics ||= client_label_values || [] - end - - def matched_metrics(project) - result = Gitlab::Prometheus::MetricGroup.for_project(project).map do |group| - group.metrics.select! do |metric| - metric.required_metrics.all?(&available_metrics.method(:include?)) - end - group - end - - result.select { |group| group.metrics.any? } - end - - def common_query_context(environment, timeframe_start:, timeframe_end:) - base_query_context(timeframe_start, timeframe_end) - .merge(QueryVariables.call(environment)) - end - - def base_query_context(timeframe_start, timeframe_end) - { - timeframe_start: timeframe_start, - timeframe_end: timeframe_end - } - end - end - end - end -end - -Gitlab::Prometheus::Queries::QueryAdditionalMetrics.prepend_mod_with('Gitlab::Prometheus::Queries::QueryAdditionalMetrics') diff --git a/lib/gitlab/prometheus/queries/validate_query.rb b/lib/gitlab/prometheus/queries/validate_query.rb deleted file mode 100644 index 160db7d44bc..00000000000 --- a/lib/gitlab/prometheus/queries/validate_query.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - module Queries - class ValidateQuery < BaseQuery - def query(query) - client_query(query) - { valid: true } - rescue Gitlab::PrometheusClient::QueryError, Gitlab::PrometheusClient::ConnectionError => ex - { valid: false, error: ex.message } - end - - def self.transform_reactive_result(result) - result[:query] = result.delete :data - result - end - end - end - end -end diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb deleted file mode 100644 index 6a6e5c22d63..00000000000 --- a/lib/gitlab/prometheus/query_variables.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - module QueryVariables - # start_time and end_time should be Time objects. - def self.call(environment, start_time: nil, end_time: nil) - { - __range: range(start_time, end_time), - ci_environment_slug: environment.slug, - kube_namespace: environment.deployment_namespace || '', - environment_filter: %(container_name!="POD",environment="#{environment.slug}"), - ci_project_name: environment.project.name, - ci_project_namespace: environment.project.namespace.name, - ci_project_path: environment.project.full_path, - ci_environment_name: environment.name - } - end - - private - - def self.range(start_time, end_time) - if start_time && end_time - range_seconds = (end_time - start_time).to_i - "#{range_seconds}s" - end - end - private_class_method :range - end - end -end diff --git a/lib/gitlab/puma/error_handler.rb b/lib/gitlab/puma/error_handler.rb new file mode 100644 index 00000000000..4efc4866431 --- /dev/null +++ b/lib/gitlab/puma/error_handler.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Puma + class ErrorHandler + PROD_ERROR_MESSAGE = "An error has occurred and reported in the system's low-level error handler." + DEV_ERROR_MESSAGE = <<~MSG + Server Error: An error has been caught by Puma's low-level error handler. + Read the Puma section of the troubleshooting docs for next steps - https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/troubleshooting/index.md#puma. + MSG + + def initialize(is_production) + @is_production = is_production + end + + def execute(ex, env, status_code) + # Puma v6.4.0 added the status_code argument in + # https://github.com/puma/puma/pull/3094 + status_code ||= 500 + + if Raven.configuration.capture_allowed? + Raven.capture_exception(ex, tags: { handler: 'puma_low_level' }, + extra: { puma_env: env, status_code: status_code }) + end + + # note the below is just a Rack response + [status_code, {}, message] + end + + private + + def message + if @is_production + PROD_ERROR_MESSAGE + else + DEV_ERROR_MESSAGE + end + end + end + end +end diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 8a604c7d8a6..4471d21b9ac 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -39,7 +39,7 @@ module Gitlab mr: :merge_request }).freeze - OPTION_MATCHER = /(?<namespace>[^\.]+)\.(?<key>[^=]+)=?(?<value>.*)/.freeze + OPTION_MATCHER = /(?<namespace>[^\.]+)\.(?<key>[^=]+)=?(?<value>.*)/ CI_SKIP = 'ci.skip' diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index f44e5383b4f..b9faf05391a 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -68,7 +68,7 @@ module Gitlab GEO_NODES_LOAD = 'SELECT 1 AS one FROM "geo_nodes" LIMIT 1' LICENSES_LOAD = 'SELECT "licenses".* FROM "licenses" ORDER BY "licenses"."id"' - SCHEMA_INTROSPECTION = %r{SELECT.*(FROM|JOIN) (pg_attribute|pg_class)}m.freeze + SCHEMA_INTROSPECTION = %r{SELECT.*(FROM|JOIN) (pg_attribute|pg_class)}m # queries can be safely ignored if they are amoritized in regular usage # (i.e. only requested occasionally and otherwise cached). diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 015dbe7063c..5cf79db83af 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -20,7 +20,7 @@ module Gitlab .+? \n```$ ) - }mix.freeze + }mix INLINE_CODE_REGEX = %r{ (?<inline_code> @@ -31,7 +31,7 @@ module Gitlab `.+?` ) - }mix.freeze + }mix HTML_BLOCK_REGEX = %r{ (?<html> @@ -44,7 +44,7 @@ module Gitlab .+? \n<\/[^>]+?>$ ) - }mix.freeze + }mix QUOTE_BLOCK_REGEX = %r{ (?<html> @@ -57,11 +57,11 @@ module Gitlab .+? \n>>>$ ) - }mix.freeze + }mix EXCLUSION_REGEX = %r{ #{CODE_REGEX} | #{INLINE_CODE_REGEX} | #{HTML_BLOCK_REGEX} | #{QUOTE_BLOCK_REGEX} - }mix.freeze + }mix attr_reader :command_definitions, :keep_actions diff --git a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb index 3794f2f8818..a8dd4aa17c5 100644 --- a/lib/gitlab/quick_actions/spend_time_and_date_separator.rb +++ b/lib/gitlab/quick_actions/spend_time_and_date_separator.rb @@ -11,7 +11,7 @@ module Gitlab # if date doesn't present return time with current date # in other cases return nil class SpendTimeAndDateSeparator - DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})}.freeze + DATE_REGEX = %r{(\d{2,4}[/\-.]\d{1,2}[/\-.]\d{1,2})} def initialize(spend_command_arg) @spend_arg = spend_command_arg diff --git a/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb b/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb index e8002656ff5..2eca87347b1 100644 --- a/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb +++ b/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb @@ -3,9 +3,9 @@ module Gitlab module QuickActions class TimelineTextAndDateTimeSeparator - DATETIME_REGEX = %r{(\d{2,4}[\-.]\d{1,2}[\-.]\d{1,2} \d{1,2}:\d{2})}.freeze - MIXED_DELIMITER = %r{([/.])}.freeze - TIME_REGEX = %r{(\d{1,2}:\d{2})}.freeze + DATETIME_REGEX = %r{(\d{2,4}[\-.]\d{1,2}[\-.]\d{1,2} \d{1,2}:\d{2})} + MIXED_DELIMITER = %r{([/.])} + TIME_REGEX = %r{(\d{1,2}:\d{2})} def initialize(timeline_event_arg) @timeline_event_arg = timeline_event_arg diff --git a/lib/gitlab/quick_actions/work_item_actions.rb b/lib/gitlab/quick_actions/work_item_actions.rb index 0a96d502862..2adee0f9a9a 100644 --- a/lib/gitlab/quick_actions/work_item_actions.rb +++ b/lib/gitlab/quick_actions/work_item_actions.rb @@ -27,6 +27,30 @@ module Gitlab command :promote_to do |type_name| @execution_message[:promote_to] = update_type(type_name, :promote_to) end + + desc { _('Change work item parent') } + explanation do |parent_param| + format(_("Change work item's parent to %{parent_ref}."), parent_ref: parent_param) + end + types WorkItem + params 'Parent #iid, reference or URL' + condition { supports_parent? && can_admin_link? } + command :set_parent do |parent_param| + @updates[:set_parent] = extract_work_items(parent_param).first + @execution_message[:set_parent] = success_msg[:set_parent] + end + + desc { _('Add children to work item') } + explanation do |child_param| + format(_("Add %{child_ref} to this work item as child(ren)."), child_ref: child_param) + end + types WorkItem + params 'Children #iids, references or URLs' + condition { supports_children? && can_admin_link? } + command :add_child do |child_param| + @updates[:add_child] = extract_work_items(child_param) + @execution_message[:add_child] = success_msg[:add_child] + end end private @@ -52,6 +76,18 @@ module Gitlab nil end + # rubocop: disable CodeReuse/ActiveRecord + def extract_work_items(params) + return if params.nil? + + issuable_type = params.include?('work_items') ? :work_item : :issue + issuables = extract_references(params, issuable_type) + return unless issuables + + WorkItem.find(issuables.pluck(:id)) + end + # rubocop: enable CodeReuse/ActiveRecord + def validate_promote_to(type) return error_msg(:not_found, action: 'promote') unless type && supports_promote_to?(type.name) return if current_user.can?(:"create_#{type.base_type}", quick_action_target) @@ -78,8 +114,8 @@ module Gitlab def error_msg(reason, action: 'convert') message = { not_found: 'Provided type is not supported', - same_type: 'Types are the same', - forbidden: 'You have insufficient permissions' + forbidden: 'You have insufficient permissions', + same_type: 'Types are the same' }.freeze format(_("Failed to %{action} this work item: %{reason}."), { action: action, reason: message[reason] }) @@ -88,9 +124,23 @@ module Gitlab def success_msg { type: _('Type changed successfully.'), - promote_to: _("Work item promoted successfully.") + promote_to: _("Work item promoted successfully."), + set_parent: _('Work item parent set successfully'), + add_child: _('Child work item(s) added successfully') } end + + def supports_parent? + ::WorkItems::HierarchyRestriction.find_by_child_type_id(quick_action_target.work_item_type_id).present? + end + + def supports_children? + ::WorkItems::HierarchyRestriction.find_by_parent_type_id(quick_action_target.work_item_type_id).present? + end + + def can_admin_link? + current_user.can?(:admin_issue_link, quick_action_target) + end end end end diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index a03116f5bb2..e45782b8be0 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -5,8 +5,9 @@ module Gitlab module Request include ::Gitlab::Utils::StrongMemoize - FILES_PATH_REGEX = %r{^/api/v\d+/projects/[^/]+/repository/files/.+}.freeze - GROUP_PATH_REGEX = %r{^/api/v\d+/groups/[^/]+/?$}.freeze + API_PATH_REGEX = %r{^/api/|/oauth/} + FILES_PATH_REGEX = %r{^/api/v\d+/projects/[^/]+/repository/files/.+} + GROUP_PATH_REGEX = %r{^/api/v\d+/groups/[^/]+/?$} def unauthenticated? !(authenticated_identifier([:api, :rss, :ics]) || authenticated_runner_id) @@ -32,7 +33,11 @@ module Gitlab end def api_request? - logical_path.start_with?('/api') + if ::Feature.enabled?(:rate_limit_oauth_api, ::Feature.current_request) + matches?(API_PATH_REGEX) + else + logical_path.start_with?('/api') + end end def logical_path diff --git a/lib/gitlab/redis/hll.rb b/lib/gitlab/redis/hll.rb index 4d1855e4637..ab144b30796 100644 --- a/lib/gitlab/redis/hll.rb +++ b/lib/gitlab/redis/hll.rb @@ -5,7 +5,7 @@ module Gitlab module Redis class HLL BATCH_SIZE = 300 - KEY_REGEX = %r{\A(\w|-|:)*\{\w*\}(\w|-|:)*\z}.freeze + KEY_REGEX = %r{\A(\w|-|:)*\{\w*\}(\w|-|:)*\z} KeyFormatError = Class.new(StandardError) def self.count(params) diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index 9ce030c0bbe..bbe5a8add4b 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -248,6 +248,19 @@ module Gitlab end end + # connection_pool gem calls `#close` method: + # + # https://github.com/mperham/connection_pool/blob/v2.4.1/lib/connection_pool.rb#L63 + # + # Let's define it explicitly instead of propagating it to method_missing + def close + if use_primary_and_secondary_stores? + [primary_store, secondary_store].map(&:close).first + else + default_store.close + end + end + private # @return [Boolean] diff --git a/lib/gitlab/redis/queues_metadata.rb b/lib/gitlab/redis/queues_metadata.rb index bb83e7709e1..a0344c93ae4 100644 --- a/lib/gitlab/redis/queues_metadata.rb +++ b/lib/gitlab/redis/queues_metadata.rb @@ -7,15 +7,6 @@ module Gitlab def config_fallback Queues end - - private - - def redis - primary_store = ::Redis.new(params) - secondary_store = ::Redis.new(config_fallback.params) - - MultiStore.new(primary_store, secondary_store, name.demodulize) - end end end end diff --git a/lib/gitlab/redis/workhorse.rb b/lib/gitlab/redis/workhorse.rb index ea0fca515fe..382797bc72a 100644 --- a/lib/gitlab/redis/workhorse.rb +++ b/lib/gitlab/redis/workhorse.rb @@ -7,15 +7,6 @@ module Gitlab def config_fallback SharedState end - - private - - def redis - primary_store = ::Redis.new(params) - secondary_store = ::Redis.new(config_fallback.params) - - MultiStore.new(primary_store, secondary_store, store_name) - end end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 1ec8818d3f5..2bcf4769b5a 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -167,20 +167,24 @@ module Gitlab cert_file = config[:ssl_params].delete(:cert_file) key_file = config[:ssl_params].delete(:key_file) - unless ::File.exist?(cert_file) - raise InvalidPathError, - "Certificate file #{cert_file} specified in in `resque.yml` does not exist." + if cert_file + unless ::File.exist?(cert_file) + raise InvalidPathError, + "Certificate file #{cert_file} specified in in `resque.yml` does not exist." + end + + config[:ssl_params][:cert] = OpenSSL::X509::Certificate.new(File.read(cert_file)) end - config[:ssl_params][:cert] = OpenSSL::X509::Certificate.new(File.read(cert_file)) + if key_file + unless ::File.exist?(key_file) + raise InvalidPathError, + "Key file #{key_file} specified in in `resque.yml` does not exist." + end - unless ::File.exist?(key_file) - raise InvalidPathError, - "Key file #{key_file} specified in in `resque.yml` does not exist." + config[:ssl_params][:key] = OpenSSL::PKey.read(File.read(key_file)) end - config[:ssl_params][:key] = OpenSSL::PKey.read(File.read(key_file)) - config end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 8ef455efe07..2fd9dc9fa09 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -6,26 +6,11 @@ module Gitlab extend MergeRequests extend Packages - def group_path_regex - # This regexp validates the string conforms to rules for a group slug: - # i.e does not start with a non-alphanumeric character except for periods or underscores, - # contains only alphanumeric characters, periods, and underscores, - # does not end with a period or forward slash, and has no leading or trailing forward slashes - # eg 'destination-path' or 'destination_pth' not 'example/com/destination/full/path' - @group_path_regex ||= %r{\A[.]?[^\W]([.]?[0-9a-z][-_]*)+\z}i - end - - def group_path_regex_message - "cannot start with a non-alphanumeric character except for periods or underscores, " \ - "can contain only alphanumeric characters, periods, and underscores, " \ - "cannot end with a period or forward slash, and has no leading or trailing forward slashes." \ - end - def project_name_regex # The character range \p{Alnum} overlaps with \u{00A9}-\u{1f9ff} # hence the Ruby warning. # https://gitlab.com/gitlab-org/gitlab/merge_requests/23165#not-easy-fixable - @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{002B}\u{00A9}-\u{1f9ff}_\. ]*\z/.freeze + @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{002B}\u{00A9}-\u{1f9ff}_\. ]*\z/ end def project_name_regex_message @@ -35,7 +20,7 @@ module Gitlab # Project path must conform to this regex. See https://gitlab.com/gitlab-org/gitlab/-/issues/27483 def oci_repository_path_regex - @oci_repository_path_regex ||= %r{\A[a-zA-Z0-9]+([._-][a-zA-Z0-9]+)*\z}.freeze + @oci_repository_path_regex ||= %r{\A[a-zA-Z0-9]+([._-][a-zA-Z0-9]+)*\z} end def oci_repository_path_regex_message @@ -43,11 +28,11 @@ module Gitlab end def group_name_regex - @group_name_regex ||= /\A#{group_name_regex_chars}\z/.freeze + @group_name_regex ||= /\A#{group_name_regex_chars}\z/ end def group_name_regex_chars - @group_name_regex_chars ||= /[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9ff}_()\. ]*/.freeze + @group_name_regex_chars ||= /[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9ff}_()\. ]*/ end def group_name_regex_message @@ -81,7 +66,7 @@ module Gitlab end def environment_name_regex - @environment_name_regex ||= /\A[#{environment_name_regex_chars_without_slash}]([#{environment_name_regex_chars}]*[#{environment_name_regex_chars_without_slash}])?\z/.freeze + @environment_name_regex ||= /\A[#{environment_name_regex_chars_without_slash}]([#{environment_name_regex_chars}]*[#{environment_name_regex_chars_without_slash}])?\z/ end def environment_name_regex_message @@ -93,7 +78,7 @@ module Gitlab end def environment_scope_regex - @environment_scope_regex ||= /\A[#{environment_scope_regex_chars}]+\z/.freeze + @environment_scope_regex ||= /\A[#{environment_scope_regex_chars}]+\z/ end def environment_scope_regex_message @@ -125,7 +110,7 @@ module Gitlab end def environment_slug_regex - @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/.freeze + @environment_slug_regex ||= /\A[a-z]([a-z0-9-]*[a-z0-9])?\z/ end def environment_slug_regex_message @@ -153,7 +138,7 @@ module Gitlab #{logs_section_prefix_regex} #{logs_section_options_regex} #{logs_section_suffix_regex} - }x.freeze + }x end MARKDOWN_CODE_BLOCK_REGEX = %r{ @@ -167,7 +152,7 @@ module Gitlab .+? \n```\ *$ ) - }mx.freeze + }mx # Code blocks: # ``` @@ -178,7 +163,7 @@ module Gitlab '^```.*?\n' \ '(?:\n|.)*?' \ '\n```\ *$' \ - ')'.freeze + ')' MARKDOWN_HTML_BLOCK_REGEX = %r{ (?<html> @@ -191,7 +176,7 @@ module Gitlab .+? \n<\/[^>]+?>\ *$ ) - }mx.freeze + }mx # HTML block: # <tag> @@ -202,28 +187,28 @@ module Gitlab '^<[^>]+?>\ *\n' \ '(?:\n|.)*?' \ '\n<\/[^>]+?>\ *$' \ - ')'.freeze + ')' # HTML comment line: # <!-- some commented text --> MARKDOWN_HTML_COMMENT_LINE_REGEX_UNTRUSTED = '(?P<html_comment_line>' \ '^<!--\ .*?\ -->\ *$' \ - ')'.freeze + ')' MARKDOWN_HTML_COMMENT_BLOCK_REGEX_UNTRUSTED = '(?P<html_comment_block>' \ '^<!--.*?\n' \ '(?:\n|.)*?' \ '\n.*?-->\ *$' \ - ')'.freeze + ')' def markdown_code_or_html_blocks @markdown_code_or_html_blocks ||= %r{ #{MARKDOWN_CODE_BLOCK_REGEX} | #{MARKDOWN_HTML_BLOCK_REGEX} - }mx.freeze + }mx end def markdown_code_or_html_blocks_untrusted @@ -292,7 +277,7 @@ module Gitlab end def utc_date_regex - @utc_date_regex ||= /\A[0-9]{4}-[0-9]{2}-[0-9]{2}\z/.freeze + @utc_date_regex ||= /\A[0-9]{4}-[0-9]{2}-[0-9]{2}\z/ end def issue @@ -304,7 +289,7 @@ module Gitlab end def base64_regex - @base64_regex ||= %r{(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?}.freeze + @base64_regex ||= %r{(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?} end def feature_flag_regex @@ -317,7 +302,7 @@ module Gitlab end def x509_subject_key_identifier_regex - @x509_subject_key_identifier_regex ||= /\A(?:\h{2}:)*\h{2}\z/.freeze + @x509_subject_key_identifier_regex ||= /\A(?:\h{2}:)*\h{2}\z/ end def ml_model_name_regex diff --git a/lib/gitlab/regex/packages.rb b/lib/gitlab/regex/packages.rb index 107f2070801..6b178933a25 100644 --- a/lib/gitlab/regex/packages.rb +++ b/lib/gitlab/regex/packages.rb @@ -9,34 +9,34 @@ module Gitlab PYPI_NORMALIZED_NAME_REGEX_STRING = '[-_.]+' # see https://github.com/apache/maven/blob/c1dfb947b509e195c75d4275a113598cf3063c3e/maven-artifact/src/main/java/org/apache/maven/artifact/Artifact.java#L46 - MAVEN_SNAPSHOT_DYNAMIC_PARTS = /\A.{0,1000}(-\d{8}\.\d{6}-\d+).{0,1000}\z/.freeze + MAVEN_SNAPSHOT_DYNAMIC_PARTS = /\A.{0,1000}(-\d{8}\.\d{6}-\d+).{0,1000}\z/ - API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+}.freeze + API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+} def conan_package_reference_regex - @conan_package_reference_regex ||= %r{\A[A-Za-z0-9]+\z}.freeze + @conan_package_reference_regex ||= %r{\A[A-Za-z0-9]+\z} end def conan_revision_regex - @conan_revision_regex ||= %r{\A0\z}.freeze + @conan_revision_regex ||= %r{\A0\z} end def conan_recipe_user_channel_regex - %r{\A(_|#{conan_name_regex})\z}.freeze + %r{\A(_|#{conan_name_regex})\z} end def conan_recipe_component_regex # https://docs.conan.io/en/latest/reference/conanfile/attributes.html#name - @conan_recipe_component_regex ||= %r{\A#{conan_name_regex}\z}.freeze + @conan_recipe_component_regex ||= %r{\A#{conan_name_regex}\z} end def composer_package_version_regex # see https://github.com/composer/semver/blob/31f3ea725711245195f62e54ffa402d8ef2fdba9/src/VersionParser.php#L215 - @composer_package_version_regex ||= %r{\Av?((\d++)(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?)?\z}.freeze + @composer_package_version_regex ||= %r{\Av?((\d++)(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?(\.(?:\d++|[xX*]))?)?\z} end def composer_dev_version_regex - @composer_dev_version_regex ||= %r{(^dev-)|(-dev$)}.freeze + @composer_dev_version_regex ||= %r{(^dev-)|(-dev$)} end def package_name_regex @@ -51,23 +51,23 @@ module Gitlab (([\w\-\.\+]*)\/)*([\w\-\.]*) ) \z - }x.freeze + }x end def maven_file_name_regex - @maven_file_name_regex ||= %r{\A[A-Za-z0-9\.\_\-\+]+\z}.freeze + @maven_file_name_regex ||= %r{\A[A-Za-z0-9\.\_\-\+]+\z} end def maven_path_regex - @maven_path_regex ||= %r{\A\@?(([\w\-\.]*)/)*([\w\-\.\+]*)\z}.freeze + @maven_path_regex ||= %r{\A\@?(([\w\-\.]*)/)*([\w\-\.\+]*)\z} end def maven_app_name_regex - @maven_app_name_regex ||= /\A[\w\-\.]+\z/.freeze + @maven_app_name_regex ||= /\A[\w\-\.]+\z/ end def maven_version_regex - @maven_version_regex ||= /\A(?!.*\.\.)[\w+.-]+\z/.freeze + @maven_version_regex ||= /\A(?!.*\.\.)[\w+.-]+\z/ end def maven_app_group_regex @@ -83,7 +83,7 @@ module Gitlab end def nuget_package_name_regex - @nuget_package_name_regex ||= %r{\A[-+\.\_a-zA-Z0-9]+\z}.freeze + @nuget_package_name_regex ||= %r{\A[-+\.\_a-zA-Z0-9]+\z} end def nuget_version_regex @@ -93,11 +93,11 @@ module Gitlab (\.#{_semver_patch_regex})? (\.\d*)? #{_semver_prerelease_build_regex}\z - /x.freeze + /x end def terraform_module_package_name_regex - @terraform_module_package_name_regex ||= %r{\A[-a-z0-9]+\/[-a-z0-9]+\z}.freeze + @terraform_module_package_name_regex ||= %r{\A[-a-z0-9]+\/[-a-z0-9]+\z} end def pypi_version_regex @@ -112,7 +112,7 @@ module Gitlab ((?:-([0-9]+))|(?:[-_\.]?(post|rev|r)[-_\.]?([0-9]+)?))? (?# post release) ([-_\.]?(dev)[-_\.]?([0-9]+)?)? (?# dev release) (?:\+([a-z0-9]+(?:[-_\.][a-z0-9]+)*))? (?# local version) - )\z}xi.freeze + )\z}xi end def debian_package_name_regex @@ -121,7 +121,7 @@ module Gitlab # @debian_package_name_regex ||= %r{\A[a-z0-9][-+\._a-z0-9]*\z}i.freeze # But we prefer a more strict version from Lintian # https://salsa.debian.org/lintian/lintian/-/blob/5080c0068ffc4a9ddee92022a91d0c2ff53e56d1/lib/Lintian/Util.pm#L116 - @debian_package_name_regex ||= %r{\A[a-z0-9][-+\.a-z0-9]+\z}.freeze + @debian_package_name_regex ||= %r{\A[a-z0-9][-+\.a-z0-9]+\z} end def debian_version_regex @@ -132,33 +132,33 @@ module Gitlab ([0-9][0-9a-z\.+~]*) (?# version) (-[0-9a-z\.+~]+){0,14} (?# -revision) (?<!-) - )\z}xi.freeze + )\z}xi end def debian_architecture_regex # See official parser: https://git.dpkg.org/cgit/dpkg/dpkg.git/tree/lib/dpkg/arch.c?id=9e0c88ec09475f4d1addde9cdba1ad7849720356#n43 # But we limit to lower case - @debian_architecture_regex ||= %r{\A#{::Packages::Debian::ARCHITECTURE_REGEX}\z}o.freeze + @debian_architecture_regex ||= %r{\A#{::Packages::Debian::ARCHITECTURE_REGEX}\z}o end def debian_distribution_regex - @debian_distribution_regex ||= %r{\A#{::Packages::Debian::DISTRIBUTION_REGEX}\z}io.freeze + @debian_distribution_regex ||= %r{\A#{::Packages::Debian::DISTRIBUTION_REGEX}\z}io end def debian_component_regex - @debian_component_regex ||= %r{\A#{::Packages::Debian::COMPONENT_REGEX}\z}o.freeze + @debian_component_regex ||= %r{\A#{::Packages::Debian::COMPONENT_REGEX}\z}o end def debian_direct_upload_filename_regex - @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb|ddeb)\z}o.freeze + @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb|ddeb)\z}o end def helm_channel_regex - @helm_channel_regex ||= %r{\A([a-zA-Z0-9](\.|-|_)?){1,255}(?<!\.|-|_)\z}.freeze + @helm_channel_regex ||= %r{\A([a-zA-Z0-9](\.|-|_)?){1,255}(?<!\.|-|_)\z} end def helm_package_regex - @helm_package_regex ||= %r{#{helm_channel_regex}}.freeze + @helm_package_regex ||= %r{#{helm_channel_regex}} end def helm_version_regex @@ -174,7 +174,7 @@ module Gitlab # only partially match "v0.0.0-20201230123456-abcdefabcdef". @unbounded_semver_regex ||= / #{_semver_major_minor_patch_regex}#{_semver_prerelease_build_regex} - /x.freeze + /x end def semver_regex @@ -190,32 +190,32 @@ module Gitlab def _semver_major_minor_patch_regex @_semver_major_minor_patch_regex ||= / #{_semver_major_regex}\.#{_semver_minor_regex}\.#{_semver_patch_regex} - /x.freeze + /x end def _semver_major_regex @_semver_major_regex ||= / (?<major>0|[1-9]\d*) - /x.freeze + /x end def _semver_minor_regex @_semver_minor_regex ||= / (?<minor>0|[1-9]\d*) - /x.freeze + /x end def _semver_patch_regex @_semver_patch_regex ||= / (?<patch>0|[1-9]\d*) - /x.freeze + /x end def _semver_prerelease_build_regex @_semver_prerelease_build_regex ||= / (?:-(?<prerelease>(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0)(?:\.(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0))*))? (?:\+(?<build>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))? - /x.freeze + /x end def prefixed_semver_regex @@ -240,7 +240,7 @@ module Gitlab | %[0-9a-f]{2})* (?# URL encoded character) )? (?# path) \b (?# word boundary) - }ix.freeze + }ix end def generic_package_version_regex @@ -256,17 +256,17 @@ module Gitlab end def sha256_regex - @sha256_regex ||= /\A[0-9a-f]{64}\z/i.freeze + @sha256_regex ||= /\A[0-9a-f]{64}\z/i end def slack_link_regex - @slack_link_regex ||= /<(.*[|].*)>/i.freeze + @slack_link_regex ||= /<(.*[|].*)>/i end private def conan_name_regex - @conan_name_regex ||= %r{[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{1,49}}.freeze + @conan_name_regex ||= %r{[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{1,49}} end end end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index d5e80053772..3a389d3363f 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -6,8 +6,7 @@ module Gitlab module RequestForgeryProtection - # rubocop:disable Rails/ApplicationController - class Controller < ActionController::Base + class Controller < BaseActionController protect_from_forgery with: :exception, prepend: true def initialize @@ -40,6 +39,5 @@ module Gitlab rescue ActionController::InvalidAuthenticityToken false end - # rubocop:enable Rails/ApplicationController end end diff --git a/lib/gitlab/robots_txt/parser.rb b/lib/gitlab/robots_txt/parser.rb index 604d2f9b35b..82505ff031a 100644 --- a/lib/gitlab/robots_txt/parser.rb +++ b/lib/gitlab/robots_txt/parser.rb @@ -3,8 +3,8 @@ module Gitlab module RobotsTxt class Parser - DISALLOW_REGEX = /^disallow: /i.freeze - ALLOW_REGEX = /^allow: /i.freeze + DISALLOW_REGEX = /^disallow: /i + ALLOW_REGEX = /^allow: /i attr_reader :disallow_rules, :allow_rules diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index f74f1489405..269fb74ceca 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -78,16 +78,16 @@ module Gitlab def puma_in_clustered_mode? return unless puma? - return unless Puma.respond_to?(:cli_config) + return unless ::Puma.respond_to?(:cli_config) - Puma.cli_config.options[:workers].to_i > 0 + ::Puma.cli_config.options[:workers].to_i > 0 end def max_threads threads = 1 # main thread - if puma? && Puma.respond_to?(:cli_config) - threads += Puma.cli_config.options[:max_threads] + if puma? && ::Puma.respond_to?(:cli_config) + threads += ::Puma.cli_config.options[:max_threads] elsif sidekiq? # 2 extra threads for the pollers in Sidekiq and Sidekiq Cron: # https://github.com/ondrejbartas/sidekiq-cron#under-the-hood diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 35e01101b3b..d06f414bd9a 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -238,9 +238,7 @@ module Gitlab def filter_milestones_by_project(milestones) candidate_project_ids = project_ids_relation - if Feature.enabled?(:search_milestones_hide_archived_projects, current_user) && !filters[:include_archived] - candidate_project_ids = candidate_project_ids.non_archived - end + candidate_project_ids = candidate_project_ids.non_archived unless filters[:include_archived] project_ids = milestones.of_projects(candidate_project_ids).select(:project_id).distinct.pluck(:project_id) # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index ba822955133..15facc4bb2f 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -14,11 +14,6 @@ module Gitlab class Shell Error = Class.new(StandardError) - PERMITTED_ACTIONS = %w[ - mv_repository remove_repository add_namespace rm_namespace mv_namespace - repository_exists? - ].freeze - class << self # Retrieve GitLab Shell secret token # @@ -80,105 +75,6 @@ module Gitlab end end - # Move or rename a repository - # - # @example Move/rename a repository - # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") - # - # @param [String] storage project's storage path - # @param [String] disk_path current project path on disk - # @param [String] new_disk_path new project path on disk - # @return [Boolean] whether repository could be moved/renamed on disk - # - # @deprecated - def mv_repository(storage, disk_path, new_disk_path) - return false if disk_path.empty? || new_disk_path.empty? - - Gitlab::Git::Repository.new(storage, "#{disk_path}.git", nil, nil).rename("#{new_disk_path}.git") - - true - rescue StandardError => e - Gitlab::ErrorTracking.track_exception(e, path: disk_path, new_path: new_disk_path, storage: storage) - - false - end - - # Removes a repository from file system, using rm_diretory which is an alias - # for rm_namespace. Given the underlying implementation removes the name - # passed as second argument on the passed storage. - # - # @example Remove a repository - # remove_repository("/path/to/storage", "gitlab/gitlab-ci") - # - # @param [String] storage project's storage path - # @param [String] disk_path current project path on disk - # - # @deprecated - def remove_repository(storage, disk_path) - return false if disk_path.empty? - - Gitlab::Git::Repository.new(storage, "#{disk_path}.git", nil, nil).remove - - true - rescue StandardError => e - Gitlab::AppLogger.warn("Repository does not exist: #{e} at: #{disk_path}.git") - Gitlab::ErrorTracking.track_exception(e, path: disk_path, storage: storage) - - false - end - - # Add empty directory for storing repositories - # - # @example Add new namespace directory - # add_namespace("default", "gitlab") - # - # @param [String] storage project's storage path - # @param [String] name namespace name - # - # @deprecated - def add_namespace(storage, name) - Gitlab::GitalyClient.allow_n_plus_1_calls do - Gitlab::GitalyClient::NamespaceService.new(storage).add(name) - end - rescue GRPC::InvalidArgument => e - raise ArgumentError, e.message - end - - # Remove directory from repositories storage - # Every repository inside this directory will be removed too - # - # @example Remove namespace directory - # rm_namespace("default", "gitlab") - # - # @param [String] storage project's storage path - # @param [String] name namespace name - # - # @deprecated - def rm_namespace(storage, name) - Gitlab::GitalyClient::NamespaceService.new(storage).remove(name) - rescue GRPC::InvalidArgument => e - raise ArgumentError, e.message - end - alias_method :rm_directory, :rm_namespace - - # Move namespace directory inside repositories storage - # - # @example Move/rename a namespace directory - # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") - # - # @param [String] storage project's storage path - # @param [String] old_name current namespace name - # @param [String] new_name new namespace name - # - # @deprecated - def mv_namespace(storage, old_name, new_name) - Gitlab::GitalyClient::NamespaceService.new(storage).rename(old_name, new_name) - rescue GRPC::InvalidArgument => e - Gitlab::ErrorTracking.track_exception(e, old_name: old_name, new_name: new_name, storage: storage) - - false - end - # Check if repository exists on disk # # @example Check if repository exists diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 7cc57f9497f..a1363e7b6b2 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -257,12 +257,7 @@ module Gitlab end def with_redis(&block) - if Feature.enabled?(:use_primary_and_secondary_stores_for_queues_metadata) || - Feature.enabled?(:use_primary_store_as_default_for_queues_metadata) - Gitlab::Redis::QueuesMetadata.with(&block) # rubocop:disable CodeReuse/ActiveRecord - else - Gitlab::Redis::Queues.with(&block) # rubocop:disable Cop/RedisQueueUsage, CodeReuse/ActiveRecord - end + Gitlab::Redis::QueuesMetadata.with(&block) # rubocop:disable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb b/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb index 93c3131d50e..0b1dc9c219e 100644 --- a/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb +++ b/lib/gitlab/sidekiq_middleware/extra_done_log_metadata.rb @@ -5,7 +5,7 @@ module Gitlab class ExtraDoneLogMetadata def call(worker, job, queue) yield - + ensure # We needed a way to pass state from a worker in to the # Gitlab::SidekiqLogging::StructuredLogger . Unfortunately the # StructuredLogger itself is not a middleware so cannot access the diff --git a/lib/gitlab/sidekiq_middleware/skip_jobs.rb b/lib/gitlab/sidekiq_middleware/skip_jobs.rb index 6cc394aa5f4..34ad843e8ee 100644 --- a/lib/gitlab/sidekiq_middleware/skip_jobs.rb +++ b/lib/gitlab/sidekiq_middleware/skip_jobs.rb @@ -67,6 +67,7 @@ module Gitlab # always returns true by default for all workers unless the FF is specifically disabled, e.g. during an incident Feature.enabled?( :"#{RUN_FEATURE_FLAG_PREFIX}_#{worker_class.name}", + Feature.current_request, type: :worker, default_enabled_if_undefined: true ) @@ -94,6 +95,7 @@ module Gitlab def drop_job?(worker_class) Feature.enabled?( :"#{DROP_FEATURE_FLAG_PREFIX}_#{worker_class.name}", + Feature.current_request, type: :worker, default_enabled_if_undefined: false ) diff --git a/lib/gitlab/slash_commands/run.rb b/lib/gitlab/slash_commands/run.rb index 40fd7ee4f20..c5330c551a1 100644 --- a/lib/gitlab/slash_commands/run.rb +++ b/lib/gitlab/slash_commands/run.rb @@ -13,7 +13,7 @@ module Gitlab end def self.available?(project) - Chat.available? && project.builds_enabled? + project.builds_enabled? end def self.allowed?(project, user) diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb index 26efb3b918d..c72a58d1ce0 100644 --- a/lib/gitlab/time_tracking_formatter.rb +++ b/lib/gitlab/time_tracking_formatter.rb @@ -15,12 +15,7 @@ module Gitlab seconds = begin - ChronicDuration.parse( - string, - CUSTOM_DAY_AND_MONTH_LENGTH.merge( - default_unit: 'hours', keep_zero: keep_zero, - use_complete_matcher: true - )) + ChronicDuration.parse(string, CUSTOM_DAY_AND_MONTH_LENGTH.merge(default_unit: 'hours', keep_zero: keep_zero)) rescue StandardError nil end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 57d3b3ec6f9..8f2dfce67bb 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -1,12 +1,16 @@ # frozen_string_literal: true +# +# IMPORTANT: With the new development of the 'gitlab-http' gem (https://gitlab.com/gitlab-org/gitlab/-/issues/415686), +# no additional change should be implemented in this class. This class will be removed after migrating all +# the usages to the new gem. +# + require 'resolv' require 'ipaddress' module Gitlab class UrlBlocker - BlockedUrlError = Class.new(StandardError) - DENY_ALL_REQUESTS_EXCEPT_ALLOWED_DEFAULT = proc { deny_all_requests_except_allowed_app_setting }.freeze # Result stores the validation result: @@ -77,7 +81,7 @@ module Gitlab return Result.new(uri, nil, proxy_in_use) unless enforce_address_info_retrievable?(uri, dns_rebind_protection, deny_all_requests_except_allowed) - raise BlockedUrlError, 'Host cannot be resolved or invalid' + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, 'Host cannot be resolved or invalid' end ip_address = ip_address(address_info) @@ -112,7 +116,7 @@ module Gitlab validate!(url, **kwargs) false - rescue BlockedUrlError + rescue Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError true end @@ -173,7 +177,7 @@ module Gitlab # # @param uri [Addressable::URI] # - # @raise [Gitlab::UrlBlocker::BlockedUrlError, ArgumentError] - BlockedUrlError raised if host is too long. + # @raise [Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, ArgumentError] - raised if host is too long. # # @return [Array<Addrinfo>] def get_address_info(uri) @@ -184,7 +188,7 @@ module Gitlab # Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters. raise unless error.message.include?('hostname too long') - raise BlockedUrlError, "Host is too long (maximum is 1024 characters)" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Host is too long (maximum is 1024 characters)" end def enforce_address_info_retrievable?(uri, dns_rebind_protection, deny_all_requests_except_allowed) @@ -232,7 +236,7 @@ module Gitlab netmask = IPAddr.new('100.64.0.0/10') return unless addrs_info.any? { |addr| netmask.include?(addr.ip_address) } - raise BlockedUrlError, "Requests to the shared address space are not allowed" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Requests to the shared address space are not allowed" end def get_port(uri) @@ -243,7 +247,7 @@ module Gitlab uri_str = uri.to_s sanitized_uri = ActionController::Base.helpers.sanitize(uri_str, tags: []) if sanitized_uri != uri_str - raise BlockedUrlError, 'HTML/CSS/JS tags are not allowed' + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, 'HTML/CSS/JS tags are not allowed' end end @@ -252,7 +256,7 @@ module Gitlab raise Addressable::URI::InvalidURIError if multiline_blocked?(parsed_url) end rescue Addressable::URI::InvalidURIError, URI::InvalidURIError - raise BlockedUrlError, 'URI is invalid' + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, 'URI is invalid' end def multiline_blocked?(parsed_url) @@ -271,12 +275,13 @@ module Gitlab return if port >= 1024 return if ports.include?(port) - raise BlockedUrlError, "Only allowed ports are #{ports.join(', ')}, and any over 1024" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, + "Only allowed ports are #{ports.join(', ')}, and any over 1024" end def validate_scheme(scheme, schemes) if scheme.blank? || (schemes.any? && schemes.exclude?(scheme)) - raise BlockedUrlError, "Only allowed schemes are #{schemes.join(', ')}" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Only allowed schemes are #{schemes.join(', ')}" end end @@ -284,7 +289,7 @@ module Gitlab return if value.blank? return if /\A\p{Alnum}/.match?(value) - raise BlockedUrlError, "Username needs to start with an alphanumeric character" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Username needs to start with an alphanumeric character" end def validate_hostname(value) @@ -292,13 +297,13 @@ module Gitlab return if IPAddress.valid?(value) return if /\A\p{Alnum}/.match?(value) - raise BlockedUrlError, "Hostname or IP address invalid" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Hostname or IP address invalid" end def validate_unicode_restriction(uri) return if uri.to_s.ascii_only? - raise BlockedUrlError, "URI must be ascii only #{uri.to_s.dump}" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "URI must be ascii only #{uri.to_s.dump}" end def validate_localhost(addrs_info) @@ -307,38 +312,39 @@ module Gitlab return if (local_ips & addrs_info.map(&:ip_address)).empty? - raise BlockedUrlError, "Requests to localhost are not allowed" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Requests to localhost are not allowed" end def validate_loopback(addrs_info) return unless addrs_info.any? { |addr| addr.ipv4_loopback? || addr.ipv6_loopback? } - raise BlockedUrlError, "Requests to loopback addresses are not allowed" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Requests to loopback addresses are not allowed" end def validate_local_network(addrs_info) return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? || addr.ipv6_unique_local? } - raise BlockedUrlError, "Requests to the local network are not allowed" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Requests to the local network are not allowed" end def validate_link_local(addrs_info) netmask = IPAddr.new('169.254.0.0/16') return unless addrs_info.any? { |addr| addr.ipv6_linklocal? || netmask.include?(addr.ip_address) } - raise BlockedUrlError, "Requests to the link local network are not allowed" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Requests to the link local network are not allowed" end - # Raises a BlockedUrlError if the instance is configured to deny all requests. + # Raises a Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError if the instance is configured to deny all requests. # # This should only be called after allow list checks have been made. def validate_deny_all_requests_except_allowed!(should_deny) return unless deny_all_requests_except_allowed?(should_deny) - raise BlockedUrlError, "Requests to hosts and IP addresses not on the Allow List are denied" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, + "Requests to hosts and IP addresses not on the Allow List are denied" end - # Raises a BlockedUrlError if any IP in `addrs_info` is the limited + # Raises a Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError if any IP in `addrs_info` is the limited # broadcast address. # https://datatracker.ietf.org/doc/html/rfc919#section-7 def validate_limited_broadcast_address(addrs_info) @@ -346,7 +352,7 @@ module Gitlab return if (blocked_ips & addrs_info.map(&:ip_address)).empty? - raise BlockedUrlError, "Requests to the limited broadcast address are not allowed" + raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "Requests to the limited broadcast address are not allowed" end def internal?(uri) diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index e203fb486e7..1b7dcaa5cf4 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -91,6 +91,8 @@ module Gitlab instance.merge_request_url(note.noteable, anchor: dom_id(note), **options) elsif note.for_snippet? instance.gitlab_snippet_url(note.noteable, anchor: dom_id(note), **options) + elsif note.for_abuse_report? + instance.admin_abuse_report_url(note.noteable, anchor: dom_id(note), **options) end end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 450575b7223..7252283d1b9 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -28,13 +28,7 @@ module Gitlab def to_context return unless %w[redis redis_hll].include?(data_source) - event_name = if data_source == 'redis_hll' - options[:events].first - elsif data_source == 'redis' - Gitlab::Usage::Metrics::Instrumentations::RedisMetric.new(attributes).redis_key - end - - Gitlab::Tracking::ServicePingContext.new(data_source: data_source, event: event_name) + Gitlab::Tracking::ServicePingContext.new(data_source: data_source, event: events.each_key.first) end def to_h @@ -58,18 +52,16 @@ module Gitlab end def validate! - unless skip_validation? - self.class.schemer.validate(attributes.deep_stringify_keys).each do |error| - error_message = <<~ERROR_MSG - Error type: #{error['type']} - Data: #{error['data']} - Path: #{error['data_pointer']} - Details: #{error['details']} - Metric file: #{path} - ERROR_MSG - - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(error_message)) - end + self.class.schemer.validate(attributes.deep_stringify_keys).each do |error| + error_message = <<~ERROR_MSG + Error type: #{error['type']} + Data: #{error['data']} + Path: #{error['data_pointer']} + Details: #{error['details']} + Metric file: #{path} + ERROR_MSG + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(error_message)) end end @@ -92,8 +84,7 @@ module Gitlab @paths ||= [Rails.root.join('config', 'metrics', '[^agg]*', '*.yml')] end - def definitions(skip_validation: false) - @skip_validation = skip_validation + def definitions @definitions ||= load_all! end @@ -110,7 +101,7 @@ module Gitlab end def context_for(key_path) - definitions[key_path].to_context + definitions[key_path]&.to_context end def schemer @@ -121,19 +112,6 @@ module Gitlab @metrics_yaml ||= definitions.values.map(&:to_h).map(&:deep_stringify_keys).to_yaml end - def metric_definitions_changed? - return false unless Rails.env.development? - - return false if @last_change_check && @last_change_check > 3.seconds.ago - - @last_change_check = Time.current - - last_change = Dir.glob(paths).map { |f| File.mtime(f) }.max - did_change = @last_metric_update != last_change - @last_metric_update = last_change - did_change - end - private def load_all! @@ -147,7 +125,7 @@ module Gitlab definition = YAML.safe_load(definition) definition.deep_symbolize_keys! - self.new(path, definition).tap(&:validate!).tap(&:category_to_lowercase) + self.new(path, definition).tap(&:category_to_lowercase) rescue StandardError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(InvalidError.new(e.message)) end @@ -175,15 +153,11 @@ module Gitlab attributes[method].present? || super end - def skip_validation? - !!attributes[:skip_validation] || @skip_validation - end - def events_from_new_structure events = attributes[:events] return unless events - events.to_h { |event| [event[:name], event[:unique].to_sym] } + events.to_h { |event| [event[:name], event[:unique]&.to_sym] } end def events_from_old_structure diff --git a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb index 0c102f0f386..31948e30992 100644 --- a/lib/gitlab/usage/metrics/instrumentations/base_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/base_metric.rb @@ -10,6 +10,7 @@ module Gitlab attr_reader :time_frame attr_reader :options + attr_reader :events class << self def available?(&block) @@ -26,6 +27,7 @@ module Gitlab def initialize(metric_definition) @time_frame = metric_definition.fetch(:time_frame) @options = metric_definition.fetch(:options, {}) + @events = metric_definition.fetch(:events, {}) end def instrumentation diff --git a/lib/gitlab/usage/metrics/instrumentations/container_registry_db_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/container_registry_db_enabled_metric.rb new file mode 100644 index 00000000000..f4306a3c319 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/container_registry_db_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class ContainerRegistryDbEnabledMetric < GenericMetric + value do + Gitlab::CurrentSettings.container_registry_db_enabled + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_csv_imports_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_csv_imports_metric.rb new file mode 100644 index 00000000000..291484dd22a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_csv_imports_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountCsvImportsMetric < DatabaseMetric + operation :count + + relation { ::Issues::CsvImport } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_jira_imports_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_jira_imports_metric.rb new file mode 100644 index 00000000000..7c32854bf4c --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_jira_imports_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountJiraImportsMetric < DatabaseMetric + operation :count + + relation { JiraImportState } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_packages_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_packages_metric.rb new file mode 100644 index 00000000000..61a3dba1942 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_packages_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountPackagesMetric < DatabaseMetric + operation :count + + relation { ::Packages::Package } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_projects_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_projects_metric.rb new file mode 100644 index 00000000000..3844aedf439 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_projects_metric.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountProjectsMetric < DatabaseMetric + operation :count + + start { Project.minimum(:id) } + finish { Project.maximum(:id) } + + relation { Project } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb b/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb new file mode 100644 index 00000000000..d07438f4bf7 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/total_count_metric.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + # Usage example + # + # In metric YAML definition: + # + # instrumentation_class: TotalCountMetric + # options: + # event: commit_pushed + # + class TotalCountMetric < BaseMetric + include Gitlab::UsageDataCounters::RedisCounter + + KEY_PREFIX = "{event_counters}_" + + def self.redis_key(event_name) + KEY_PREFIX + event_name + end + + def value + events.sum do |event| + redis_usage_data do + total_count(self.class.redis_key(event[:name])) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index c3378856633..b2027791e9d 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -135,15 +135,6 @@ module Gitlab } end # rubocop: enable Metrics/AbcSize - - def system_usage_data_monthly - { - counts_monthly: { - projects: count(Project.where(monthly_time_range_db_params), start: minimum_id(Project), finish: maximum_id(Project)), - packages: count(::Packages::Package.where(monthly_time_range_db_params)) - } - } - end # rubocop: enable CodeReuse/ActiveRecord def system_usage_data_license @@ -379,8 +370,6 @@ module Gitlab bulk_imports: { gitlab_v1: count(::BulkImport.where(**time_period, source_type: :gitlab)) }, - project_imports: project_imports(time_period), - issue_imports: issue_imports(time_period), group_imports: group_imports(time_period) } end @@ -498,7 +487,6 @@ module Gitlab def usage_data_metrics system_usage_data_license .merge(system_usage_data) - .merge(system_usage_data_monthly) .merge(system_usage_data_weekly) .merge(features_usage_data) .merge(components_usage_data) @@ -569,33 +557,6 @@ module Gitlab omniauth_provider_names.reject { |name| name.starts_with?('ldap') } end - def project_imports(time_period) - time_frame = metric_time_period(time_period) - counters = { - gitlab_project: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitlab_project' }), - github: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'github' }), - bitbucket: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket' }), - bitbucket_server: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket_server' }), - gitea: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitea' }), - git: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'git' }), - manifest: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'manifest' }), - gitlab_migration: add_metric('CountBulkImportsEntitiesMetric', time_frame: time_frame, options: { source_type: :project_entity }) - } - - counters[:total] = add_metric('CountImportedProjectsTotalMetric', time_frame: time_frame) - - counters - end - - def issue_imports(time_period) - time_frame = metric_time_period(time_period) - { - jira: count(::JiraImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord - fogbugz: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'fogbugz' }), - csv: count(::Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord - } - end - def group_imports(time_period) time_frame = metric_time_period(time_period) { diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index eb141a2e2f6..e0a4f879f48 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -13,11 +13,7 @@ module Gitlab::UsageDataCounters Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: project.id) namespace = project.namespace - context = Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, - event: event_name).to_context - label = 'redis_hll_counters.ci_templates.ci_templates_total_unique_counts_monthly' - Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, - project: project, context: [context], user: user, label: label) + Gitlab::InternalEvents.track_event('ci_template_included', namespace: namespace, project: project, user: user) end def ci_templates(relative_base = 'lib/gitlab/ci/templates') diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index d19dd6cd856..f2db7e3c9b9 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -260,12 +260,7 @@ module Gitlab protected def with_redis(&blk) - if Feature.enabled?(:use_primary_and_secondary_stores_for_workhorse) || - Feature.enabled?(:use_primary_store_as_default_for_workhorse) - Gitlab::Redis::Workhorse.with(&blk) # rubocop:disable CodeReuse/ActiveRecord - else - Gitlab::Redis::SharedState.with(&blk) # rubocop:disable CodeReuse/ActiveRecord - end + Gitlab::Redis::Workhorse.with(&blk) # rubocop:disable CodeReuse/ActiveRecord end # This is the outermost encoding of a senddata: header. It is safe for diff --git a/lib/product_analytics/settings.rb b/lib/product_analytics/settings.rb deleted file mode 100644 index ad03c34cdd2..00000000000 --- a/lib/product_analytics/settings.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module ProductAnalytics - class Settings - BASE_CONFIG_KEYS = %w[product_analytics_data_collector_host cube_api_base_url cube_api_key].freeze - - SNOWPLOW_CONFIG_KEYS = (%w[product_analytics_configurator_connection_string] + - BASE_CONFIG_KEYS).freeze - - ALL_CONFIG_KEYS = (ProductAnalytics::Settings::BASE_CONFIG_KEYS + - ProductAnalytics::Settings::SNOWPLOW_CONFIG_KEYS).freeze - - def initialize(project:) - @project = project - end - - def enabled? - ::Gitlab::CurrentSettings.product_analytics_enabled? && configured? - end - - def configured? - ALL_CONFIG_KEYS.all? do |key| - get_setting_value(key).present? - end - end - - ALL_CONFIG_KEYS.each do |key| - define_method key.to_sym do - get_setting_value(key) - end - end - - class << self - def for_project(project) - ProductAnalytics::Settings.new(project: project) - end - end - - private - - # rubocop:disable GitlabSecurity/PublicSend - def get_setting_value(key) - @project.project_setting.public_send(key).presence || - ::Gitlab::CurrentSettings.public_send(key) - end - # rubocop:enable GitlabSecurity/PublicSend - end -end diff --git a/lib/sidebars/admin/menus/admin_overview_menu.rb b/lib/sidebars/admin/menus/admin_overview_menu.rb index 57c9ff4dcb0..5974b4d16ae 100644 --- a/lib/sidebars/admin/menus/admin_overview_menu.rb +++ b/lib/sidebars/admin/menus/admin_overview_menu.rb @@ -28,7 +28,7 @@ module Sidebars override :extra_container_html_options def extra_container_html_options - { 'data-qa-selector': 'admin_overview_submenu_content' } + { testid: 'admin-overview-submenu-content' } end private diff --git a/lib/sidebars/admin/menus/analytics_menu.rb b/lib/sidebars/admin/menus/analytics_menu.rb index 944f7f6bba7..4bad6fa43e8 100644 --- a/lib/sidebars/admin/menus/analytics_menu.rb +++ b/lib/sidebars/admin/menus/analytics_menu.rb @@ -24,7 +24,7 @@ module Sidebars override :extra_container_html_options def extra_container_html_options - { 'data-qa-selector': 'admin_sidebar_analytics_submenu_content' } + { testid: 'admin-sidebar-analytics-submenu-content' } end private diff --git a/lib/sidebars/admin/menus/monitoring_menu.rb b/lib/sidebars/admin/menus/monitoring_menu.rb index 2cf21e1bf77..1683147958c 100644 --- a/lib/sidebars/admin/menus/monitoring_menu.rb +++ b/lib/sidebars/admin/menus/monitoring_menu.rb @@ -26,7 +26,7 @@ module Sidebars override :extra_container_html_options def extra_container_html_options - { 'data-qa-selector': 'admin_monitoring_menu_link' } + { testid: 'admin-monitoring-menu-link' } end private diff --git a/lib/sidebars/groups/menus/observability_menu.rb b/lib/sidebars/groups/menus/observability_menu.rb deleted file mode 100644 index 268528356f1..00000000000 --- a/lib/sidebars/groups/menus/observability_menu.rb +++ /dev/null @@ -1,80 +0,0 @@ -# frozen_string_literal: true - -module Sidebars - module Groups - module Menus - class ObservabilityMenu < ::Sidebars::Menu - override :configure_menu_items - def configure_menu_items - add_item(explore_menu_item) if Gitlab::Observability.allowed_for_action?(context.current_user, context.group, - :explore) - - add_item(datasources_menu_item) if Gitlab::Observability.allowed_for_action?(context.current_user, - context.group, :datasources) - end - - override :title - def title - _('Observability') - end - - override :sprite_icon - def sprite_icon - 'monitor' - end - - override :render? - def render? - Gitlab::Observability.allowed_for_action?(context.current_user, context.group, :explore) - end - - override :serialize_as_menu_item_args - def serialize_as_menu_item_args - nil - end - - private - - def dashboards_menu_item - ::Sidebars::MenuItem.new( - title: s_('Observability|Dashboards'), - link: group_observability_dashboards_path(context.group), - super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::MonitorMenu, - active_routes: { path: 'groups/observability#dashboards' }, - item_id: :dashboards - ) - end - - def explore_menu_item - ::Sidebars::MenuItem.new( - title: s_('Observability|Explore telemetry data'), - link: group_observability_explore_path(context.group), - super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::MonitorMenu, - active_routes: { path: 'groups/observability#explore' }, - item_id: :explore - ) - end - - def datasources_menu_item - ::Sidebars::MenuItem.new( - title: s_('Observability|Data sources'), - link: group_observability_datasources_path(context.group), - super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::MonitorMenu, - active_routes: { path: 'groups/observability#datasources' }, - item_id: :datasources - ) - end - - def manage_menu_item - ::Sidebars::MenuItem.new( - title: s_('Observability|Manage dashboards'), - link: group_observability_manage_path(context.group), - super_sidebar_parent: ::Sidebars::Groups::SuperSidebarMenus::MonitorMenu, - active_routes: { path: 'groups/observability#manage' }, - item_id: :manage - ) - end - end - end - end -end diff --git a/lib/sidebars/groups/panel.rb b/lib/sidebars/groups/panel.rb index 77ca51ddf92..185e49938ef 100644 --- a/lib/sidebars/groups/panel.rb +++ b/lib/sidebars/groups/panel.rb @@ -12,7 +12,6 @@ module Sidebars add_menu(Sidebars::Groups::Menus::MergeRequestsMenu.new(context)) add_menu(Sidebars::Groups::Menus::CiCdMenu.new(context)) add_menu(Sidebars::Groups::Menus::KubernetesMenu.new(context)) - add_menu(Sidebars::Groups::Menus::ObservabilityMenu.new(context)) add_menu(Sidebars::Groups::Menus::PackagesRegistriesMenu.new(context)) add_menu(Sidebars::Groups::Menus::CustomerRelationsMenu.new(context)) add_menu(Sidebars::Groups::Menus::SettingsMenu.new(context)) diff --git a/lib/sidebars/groups/super_sidebar_menus/monitor_menu.rb b/lib/sidebars/groups/super_sidebar_menus/monitor_menu.rb deleted file mode 100644 index 8ee0aaaa808..00000000000 --- a/lib/sidebars/groups/super_sidebar_menus/monitor_menu.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Sidebars - module Groups - module SuperSidebarMenus - class MonitorMenu < ::Sidebars::Menu - override :title - def title - s_('Navigation|Monitor') - end - - override :sprite_icon - def sprite_icon - 'monitor' - end - - override :configure_menu_items - def configure_menu_items - [ - :explore, - :datasources - ].each { |id| add_item(::Sidebars::NilMenuItem.new(item_id: id)) } - end - end - end - end -end diff --git a/lib/sidebars/groups/super_sidebar_panel.rb b/lib/sidebars/groups/super_sidebar_panel.rb index 01c6f88dcc0..61d1e082a4b 100644 --- a/lib/sidebars/groups/super_sidebar_panel.rb +++ b/lib/sidebars/groups/super_sidebar_panel.rb @@ -20,7 +20,6 @@ module Sidebars add_menu(Sidebars::Groups::SuperSidebarMenus::SecureMenu.new(context)) add_menu(Sidebars::Groups::SuperSidebarMenus::DeployMenu.new(context)) add_menu(Sidebars::Groups::SuperSidebarMenus::OperationsMenu.new(context)) - add_menu(Sidebars::Groups::SuperSidebarMenus::MonitorMenu.new(context)) add_menu(Sidebars::Groups::SuperSidebarMenus::AnalyzeMenu.new(context)) pick_from_old_menus(old_menus) diff --git a/lib/sidebars/organizations/menus/settings_menu.rb b/lib/sidebars/organizations/menus/settings_menu.rb new file mode 100644 index 00000000000..b26a62dca5a --- /dev/null +++ b/lib/sidebars/organizations/menus/settings_menu.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Sidebars + module Organizations + module Menus + class SettingsMenu < ::Sidebars::Menu + override :title + def title + _('Settings') + end + + override :sprite_icon + def sprite_icon + 'settings' + end + + override :pick_into_super_sidebar? + def pick_into_super_sidebar? + true + end + + override :render? + def render? + can?(context.current_user, :admin_organization, context.container) + end + + override :configure_menu_items + def configure_menu_items + add_item( + ::Sidebars::MenuItem.new( + title: _('General'), + link: general_settings_organization_path(context.container), + super_sidebar_parent: ::Sidebars::Organizations::Menus::SettingsMenu, + active_routes: { path: 'organizations/settings#general' }, + item_id: :organization_settings_general + ) + ) + end + end + end + end +end diff --git a/lib/sidebars/organizations/panel.rb b/lib/sidebars/organizations/panel.rb index 159ccb6cbe9..81e6c7af005 100644 --- a/lib/sidebars/organizations/panel.rb +++ b/lib/sidebars/organizations/panel.rb @@ -15,6 +15,7 @@ module Sidebars set_scope_menu(Sidebars::Organizations::Menus::ScopeMenu.new(context)) add_menu(Sidebars::StaticMenu.new(context)) add_menu(Sidebars::Organizations::Menus::ManageMenu.new(context)) + add_menu(Sidebars::Organizations::Menus::SettingsMenu.new(context)) end end end diff --git a/lib/sidebars/projects/menus/deployments_menu.rb b/lib/sidebars/projects/menus/deployments_menu.rb index ff2f833763a..e4e2e55333e 100644 --- a/lib/sidebars/projects/menus/deployments_menu.rb +++ b/lib/sidebars/projects/menus/deployments_menu.rb @@ -85,7 +85,7 @@ module Sidebars end def pages_menu_item - unless context.project.pages_available? && context.current_user&.can?(:update_pages, context.project) + unless ::Gitlab::Pages.enabled? && context.current_user&.can?(:update_pages, context.project) return ::Sidebars::NilMenuItem.new(item_id: :pages) end @@ -101,3 +101,5 @@ module Sidebars end end end + +Sidebars::Projects::Menus::DeploymentsMenu.prepend_mod diff --git a/lib/sidebars/projects/menus/settings_menu.rb b/lib/sidebars/projects/menus/settings_menu.rb index 142d803037b..8fed1c46425 100644 --- a/lib/sidebars/projects/menus/settings_menu.rb +++ b/lib/sidebars/projects/menus/settings_menu.rb @@ -6,19 +6,11 @@ module Sidebars class SettingsMenu < ::Sidebars::Menu override :configure_menu_items def configure_menu_items - return false unless can?(context.current_user, :admin_project, context.project) - - add_item(general_menu_item) - add_item(integrations_menu_item) - add_item(webhooks_menu_item) - add_item(access_tokens_menu_item) - add_item(repository_menu_item) - add_item(merge_requests_menu_item) - add_item(ci_cd_menu_item) - add_item(packages_and_registries_menu_item) - add_item(monitor_menu_item) - add_item(usage_quotas_menu_item) + return false if enabled_menu_items.empty? + enabled_menu_items.each do |menu_item| + add_item(menu_item) + end true end @@ -51,6 +43,29 @@ module Sidebars private + def enabled_menu_items + if can?(context.current_user, :admin_project, context.project) + [ + general_menu_item, + integrations_menu_item, + webhooks_menu_item, + access_tokens_menu_item, + repository_menu_item, + merge_requests_menu_item, + ci_cd_menu_item, + packages_and_registries_menu_item, + monitor_menu_item, + usage_quotas_menu_item + ] + elsif context.current_user && can?(context.current_user, :manage_resource_access_tokens, context.project) + [ + access_tokens_menu_item + ] + else + [] + end + end + def general_menu_item ::Sidebars::MenuItem.new( title: _('General'), diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index 1753483b091..ef9d2b5e13a 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -87,6 +87,74 @@ namespace :gitlab do end end + desc "GitLab | Cleanup | Clean missed source branches to be deleted" + task remove_missed_source_branches: :gitlab_environment do + warn_user_is_not_gitlab + + logger.info("Gitlab|Cleanup|Clean up missed source branches|Executed by #{gitlab_user}") + + if ENV['LIMIT_TO_DELETE'].present? && !ENV['LIMIT_TO_DELETE'].to_i.between?(1, 10000) + logger.info("Please specify a limit between 1 and 10000") + next + end + + if ENV['BATCH_SIZE'].present? && !ENV['BATCH_SIZE'].to_i.between?(1, 1000) + logger.info("Please specify a batch size between 1 and 1000") + next + end + + batch_size = ENV['BATCH_SIZE'].present? ? ENV['BATCH_SIZE'].to_i : 1000 + limit = ENV['LIMIT_TO_DELETE'].present? ? ENV['LIMIT_TO_DELETE'].to_i : 10000 + + project = find_project + user = User.find_by_id(ENV['USER_ID']&.to_i) + + number_deleted = 0 + + # rubocop: disable Layout/LineLength + MergeRequest + .merged + .where(project: project) + .each_batch(of: batch_size) do |mrs| + matching_mrs = mrs.where( + "merge_params LIKE '%force_remove_source_branch: ''1''%' OR merge_params LIKE '%should_remove_source_branch: ''1''%'" + ) + + branches_to_delete = [] + + # rubocop: enable Layout/LineLength + matching_mrs.each do |mr| + next unless mr.source_branch_exists? && mr.can_remove_source_branch?(user) + + # Ensuring that only this MR exists for the source branch + if MergeRequest.where(project: project).where.not(id: mr.id).where(source_branch: mr.source_branch).exists? + next + end + + latest_diff_sha = mr.latest_merge_request_diff.head_commit_sha + + next unless latest_diff_sha + + branches_to_delete << { reference: mr.source_branch_ref, old_sha: latest_diff_sha, +new_sha: Gitlab::Git::BLANK_SHA } + + break if number_deleted + branches_to_delete.size >= limit + end + + if dry_run? + logger.info "DRY RUN: Branches to be deleted in batch #{branches_to_delete.join(',')}" + logger.info "DRY RUN: Count: #{branches_to_delete.size}" + else + project.repository.raw.update_refs(branches_to_delete) + logger.info "Branches deleted #{branches_to_delete.join(',')}" + end + + number_deleted += branches_to_delete.size + + break if number_deleted >= limit + end + end + desc 'GitLab | Cleanup | Clean orphan LFS files' task orphan_lfs_files: :gitlab_environment do warn_user_is_not_gitlab diff --git a/lib/tasks/gitlab/doctor/secrets.rake b/lib/tasks/gitlab/doctor/secrets.rake index 29f0f36c705..dd005005edf 100644 --- a/lib/tasks/gitlab/doctor/secrets.rake +++ b/lib/tasks/gitlab/doctor/secrets.rake @@ -10,5 +10,21 @@ namespace :gitlab do Gitlab::Doctor::Secrets.new(logger).run! end + + desc "GitLab | Reset encrypted tokens for specific models" + task reset_encrypted_tokens: :gitlab_environment do + logger = Logger.new($stdout) + + logger.level = Gitlab::Utils.to_boolean(ENV['VERBOSE']) ? Logger::DEBUG : Logger::INFO + model_names = ENV['MODEL_NAMES']&.split(',') + token_names = ENV['TOKEN_NAMES']&.split(',') + dry_run = Gitlab::Utils.to_boolean(ENV['DRY_RUN']) + dry_run = true if dry_run.nil? + + next logger.info("No models were specified, please use MODEL_NAMES environment variable") unless model_names + next logger.info("No tokens were specified, please use TOKEN_NAMES environment variable") unless token_names + + Gitlab::Doctor::ResetTokens.new(logger, model_names: model_names, token_names: token_names, dry_run: dry_run).run! + end end end diff --git a/lib/tasks/gitlab/password.rake b/lib/tasks/gitlab/password.rake index 02c28578a2a..a7b7aafc0c9 100644 --- a/lib/tasks/gitlab/password.rake +++ b/lib/tasks/gitlab/password.rake @@ -14,6 +14,7 @@ namespace :gitlab do user.password = password user.password_confirmation = password_confirm + user.password_automatically_set = false user.send_only_admin_changed_your_password_notification! unless user.save diff --git a/lib/tasks/gitlab/storage.rake b/lib/tasks/gitlab/storage.rake deleted file mode 100644 index eb5eeed531f..00000000000 --- a/lib/tasks/gitlab/storage.rake +++ /dev/null @@ -1,186 +0,0 @@ -# frozen_string_literal: true - -namespace :gitlab do - namespace :storage do - desc 'GitLab | Storage | Migrate existing projects to Hashed Storage' - task migrate_to_hashed: :environment do - if Gitlab::Database.read_only? - abort 'This task requires database write access. Exiting.' - end - - storage_migrator = Gitlab::HashedStorage::Migrator.new - helper = Gitlab::HashedStorage::RakeHelper - - if storage_migrator.rollback_pending? - abort "There is already a rollback operation in progress, " \ - "running a migration at the same time may have unexpected consequences." - end - - if helper.range_single_item? - project = Project.with_unmigrated_storage.find_by(id: helper.range_from) - - unless project - abort "There are no projects requiring storage migration with ID=#{helper.range_from}" - end - - puts "Enqueueing storage migration of #{project.full_path} (ID=#{project.id})..." - storage_migrator.migrate(project) - else - legacy_projects_count = if helper.using_ranges? - Project.with_unmigrated_storage.id_in(helper.range_from..helper.range_to).count - else - Project.with_unmigrated_storage.count - end - - if legacy_projects_count == 0 - abort 'There are no projects requiring storage migration. Nothing to do!' - end - - print "Enqueuing migration of #{legacy_projects_count} projects in batches of #{helper.batch_size}" - - helper.project_id_batches_migration do |start, finish| - storage_migrator.bulk_schedule_migration(start: start, finish: finish) - - print '.' - end - end - - puts ' Done!' - end - - desc 'GitLab | Storage | Rollback existing projects to Legacy Storage' - task rollback_to_legacy: :environment do - if Gitlab::Database.read_only? - abort 'This task requires database write access. Exiting.' - end - - storage_migrator = Gitlab::HashedStorage::Migrator.new - helper = Gitlab::HashedStorage::RakeHelper - - if storage_migrator.migration_pending? - abort "There is already a migration operation in progress, " \ - "running a rollback at the same time may have unexpected consequences." - end - - if helper.range_single_item? - project = Project.with_storage_feature(:repository).find_by(id: helper.range_from) - - unless project - abort "There are no projects that can be rolledback with ID=#{helper.range_from}" - end - - puts "Enqueueing storage rollback of #{project.full_path} (ID=#{project.id})..." - storage_migrator.rollback(project) - else - hashed_projects_count = if helper.using_ranges? - Project.with_storage_feature(:repository).id_in(helper.range_from..helper.range_to).count - else - Project.with_storage_feature(:repository).count - end - - if hashed_projects_count == 0 - abort 'There are no projects that can have storage rolledback. Nothing to do!' - end - - print "Enqueuing rollback of #{hashed_projects_count} projects in batches of #{helper.batch_size}" - - helper.project_id_batches_rollback do |start, finish| - storage_migrator.bulk_schedule_rollback(start: start, finish: finish) - - print '.' - end - end - - puts ' Done!' - end - - desc 'Gitlab | Storage | Summary of existing projects using Legacy Storage' - task legacy_projects: :environment do - # Required to prevent Docker upgrade to 14.0 if there data on legacy storage - # See: https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5311#note_590454698 - wait_until_database_is_ready do - helper = Gitlab::HashedStorage::RakeHelper - helper.relation_summary('projects using Legacy Storage', Project.without_storage_feature(:repository)) - end - end - - desc 'Gitlab | Storage | List existing projects using Legacy Storage' - task list_legacy_projects: :environment do - helper = Gitlab::HashedStorage::RakeHelper - helper.projects_list('projects using Legacy Storage', Project.without_storage_feature(:repository)) - end - - desc 'Gitlab | Storage | Summary of existing projects using Hashed Storage' - task hashed_projects: :environment do - helper = Gitlab::HashedStorage::RakeHelper - helper.relation_summary('projects using Hashed Storage', Project.with_storage_feature(:repository)) - end - - desc 'Gitlab | Storage | List existing projects using Hashed Storage' - task list_hashed_projects: :environment do - helper = Gitlab::HashedStorage::RakeHelper - helper.projects_list('projects using Hashed Storage', Project.with_storage_feature(:repository)) - end - - desc 'Gitlab | Storage | Prune projects using Hashed Storage. Remove all hashed directories that do not have a project associated' - task prune_hashed_projects: [:environment, :gitlab_environment] do - if Rails.env.production? - abort('This destructive action may only be run in development') - end - - helper = Gitlab::HashedStorage::RakeHelper - name = 'projects using Hashed Storage' - relation = Project.with_storage_feature(:repository) - root = Gitlab.config.repositories.storages['default'].legacy_disk_path - dry_run = !ENV['FORCE'].present? - - helper.prune(name, relation, dry_run: dry_run, root: root) - end - - desc 'Gitlab | Storage | Summary of project attachments using Legacy Storage' - task legacy_attachments: :environment do - # Required to prevent Docker upgrade to 14.0 if there data on legacy storage - # See: https://gitlab.com/gitlab-org/omnibus-gitlab/-/merge_requests/5311#note_590454698 - wait_until_database_is_ready do - helper = Gitlab::HashedStorage::RakeHelper - helper.relation_summary('attachments using Legacy Storage', helper.legacy_attachments_relation) - end - end - - desc 'Gitlab | Storage | List existing project attachments using Legacy Storage' - task list_legacy_attachments: :environment do - helper = Gitlab::HashedStorage::RakeHelper - helper.attachments_list('attachments using Legacy Storage', helper.legacy_attachments_relation) - end - - desc 'Gitlab | Storage | Summary of project attachments using Hashed Storage' - task hashed_attachments: :environment do - helper = Gitlab::HashedStorage::RakeHelper - helper.relation_summary('attachments using Hashed Storage', helper.hashed_attachments_relation) - end - - desc 'Gitlab | Storage | List existing project attachments using Hashed Storage' - task list_hashed_attachments: :environment do - helper = Gitlab::HashedStorage::RakeHelper - helper.attachments_list('attachments using Hashed Storage', helper.hashed_attachments_relation) - end - - def wait_until_database_is_ready - attempts = (ENV['MAX_DATABASE_CONNECTION_CHECKS'] || 1).to_i - inverval = (ENV['MAX_DATABASE_CONNECTION_CHECK_INTERVAL'] || 10).to_f - - attempts.to_i.times do - unless ApplicationRecord.database.exists? - puts "Waiting until database is ready before continuing...".color(:yellow) - sleep inverval - end - end - - yield - rescue ActiveRecord::ConnectionNotEstablished => ex - puts "Failed to connect to the database...".color(:red) - puts "Error: #{ex}" - exit 1 - end - end -end diff --git a/lib/tasks/gitlab/tw/codeowners.rake b/lib/tasks/gitlab/tw/codeowners.rake index cea66125fd0..495d7a339b8 100644 --- a/lib/tasks/gitlab/tw/codeowners.rake +++ b/lib/tasks/gitlab/tw/codeowners.rake @@ -25,9 +25,10 @@ namespace :tw do CodeOwnerRule.new('AI Model Validation', '@sselhorn'), CodeOwnerRule.new('Analytics Instrumentation', '@lciutacu'), CodeOwnerRule.new('Anti-Abuse', '@phillipwells'), - CodeOwnerRule.new('Application Performance', '@jglassman1'), + CodeOwnerRule.new('Cloud Connector', '@jglassman1'), CodeOwnerRule.new('Authentication and Authorization', '@jglassman1'), # CodeOwnerRule.new('Billing and Subscription Management', ''), + CodeOwnerRule.new('Code Creation', '@jglassman1'), CodeOwnerRule.new('Code Review', '@aqualls'), CodeOwnerRule.new('Compliance', '@eread'), CodeOwnerRule.new('Composition Analysis', '@rdickenson'), @@ -44,7 +45,7 @@ namespace :tw do CodeOwnerRule.new('Documentation Guidelines', '@sselhorn'), CodeOwnerRule.new('Duo Chat', '@sselhorn'), CodeOwnerRule.new('Dynamic Analysis', '@rdickenson'), - CodeOwnerRule.new('IDE', '@ashrafkhamis'), + CodeOwnerRule.new('Editor Extensions', '@aqualls'), CodeOwnerRule.new('Foundations', '@sselhorn'), # CodeOwnerRule.new('Fulfillment Platform', ''), CodeOwnerRule.new('Fuzz Testing', '@rdickenson'), @@ -52,6 +53,7 @@ namespace :tw do CodeOwnerRule.new('Gitaly', '@eread'), # CodeOwnerRule.new('GitLab Dedicated', ''), CodeOwnerRule.new('Global Search', '@ashrafkhamis'), + CodeOwnerRule.new('IDE', '@ashrafkhamis'), CodeOwnerRule.new('Import and Integrate', '@eread @ashrafkhamis'), CodeOwnerRule.new('Infrastructure', '@sselhorn'), # CodeOwnerRule.new('Knowledge', ''), @@ -69,11 +71,11 @@ namespace :tw do CodeOwnerRule.new('Provision', '@fneill'), CodeOwnerRule.new('Purchase', '@fneill'), CodeOwnerRule.new('Redirect', 'Redirect'), - CodeOwnerRule.new('Respond', '@msedlakjakubowski'), + # CodeOwnerRule.new('Respond', ''), CodeOwnerRule.new('Runner', '@fneill'), CodeOwnerRule.new('Runner SaaS', '@fneill'), CodeOwnerRule.new('Security Policies', '@rdickenson'), - CodeOwnerRule.new('Source Code', ->(path) { path.start_with?('/doc/user') ? '@aqualls' : '@msedlakjakubowski' }), + CodeOwnerRule.new('Source Code', '@msedlakjakubowski'), CodeOwnerRule.new('Static Analysis', '@rdickenson'), CodeOwnerRule.new('Style Guide', '@sselhorn'), CodeOwnerRule.new('Tenant Scale', '@lciutacu'), diff --git a/lib/vs_code/settings.rb b/lib/vs_code/settings.rb new file mode 100644 index 00000000000..30b91ebb16f --- /dev/null +++ b/lib/vs_code/settings.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module VsCode + module Settings + DEFAULT_MACHINE = { + id: 1, + uuid: "3aa16b0f-652e-4850-8429-a00190dac6aa", + version: 1, + setting_type: "machines", + machines: [ + { + id: 1, + name: "GitLab WebIDE", + platform: "GitLab" + } + ] + }.freeze + SETTINGS_TYPES = %w[settings extensions globalState machines keybindings snippets tasks].freeze + DEFAULT_SESSION = "1" + NO_CONTENT_ETAG = "0" + end +end |