diff options
Diffstat (limited to 'lib/gitlab')
295 files changed, 2817 insertions, 2725 deletions
diff --git a/lib/gitlab/alert_management/payload/prometheus.rb b/lib/gitlab/alert_management/payload/prometheus.rb index 15fa91646c8..e64fb52c0cf 100644 --- a/lib/gitlab/alert_management/payload/prometheus.rb +++ b/lib/gitlab/alert_management/payload/prometheus.rb @@ -7,32 +7,32 @@ module Gitlab class Prometheus < Base extend Gitlab::Utils::Override - attribute :alert_markdown, paths: %w(annotations gitlab_incident_markdown) + attribute :alert_markdown, paths: %w[annotations gitlab_incident_markdown] attribute :annotations, paths: 'annotations' - attribute :description, paths: %w(annotations description) + attribute :description, paths: %w[annotations description] attribute :ends_at, paths: 'endsAt', type: :time - attribute :environment_name, paths: %w(labels gitlab_environment_name) - attribute :generator_url, paths: %w(generatorURL) + attribute :environment_name, paths: %w[labels gitlab_environment_name] + attribute :generator_url, paths: %w[generatorURL] attribute :gitlab_y_label, - paths: [%w(annotations gitlab_y_label), - %w(annotations title), - %w(annotations summary), - %w(labels alertname)] - attribute :runbook, paths: %w(annotations runbook) + paths: [%w[annotations gitlab_y_label], + %w[annotations title], + %w[annotations summary], + %w[labels alertname]] + attribute :runbook, paths: %w[annotations runbook] attribute :starts_at, paths: 'startsAt', type: :time, fallback: -> { Time.current.utc } attribute :status, paths: 'status' attribute :title, - paths: [%w(annotations title), - %w(annotations summary), - %w(labels alertname)] + paths: [%w[annotations title], + %w[annotations summary], + %w[labels alertname]] attribute :starts_at_raw, - paths: [%w(startsAt)] + paths: [%w[startsAt]] private :starts_at_raw - attribute :severity_raw, paths: %w(labels severity) + attribute :severity_raw, paths: %w[labels severity] private :severity_raw METRIC_TIME_WINDOW = 30.minutes diff --git a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb index b2e1c9e2379..08439ac5f1e 100644 --- a/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb +++ b/lib/gitlab/asciidoc/syntax_highlighter/html_pipeline_adapter.rb @@ -7,7 +7,7 @@ module Gitlab register_for 'gitlab-html-pipeline' def format(node, lang, opts) - %(<pre #{lang ? %[lang="#{lang}"] : ''}><code>#{node.content}</code></pre>) + %(<pre #{lang ? %(lang="#{lang}") : ''}><code>#{node.content}</code></pre>) end end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index cafa75d5f59..9ddfc995535 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -5,12 +5,19 @@ module Gitlab MissingPersonalAccessTokenError = Class.new(StandardError) IpBlocked = Class.new(StandardError) + # Scopes used for GitLab internal API (Kubernetes cluster access) + K8S_PROXY_SCOPE = :k8s_proxy + # Scopes used for GitLab API access API_SCOPE = :api READ_API_SCOPE = :read_api READ_USER_SCOPE = :read_user CREATE_RUNNER_SCOPE = :create_runner - API_SCOPES = [API_SCOPE, READ_API_SCOPE, READ_USER_SCOPE, CREATE_RUNNER_SCOPE].freeze + API_SCOPES = [API_SCOPE, READ_API_SCOPE, READ_USER_SCOPE, CREATE_RUNNER_SCOPE, K8S_PROXY_SCOPE].freeze + + # Scopes for Duo + AI_FEATURES = :ai_features + AI_FEATURES_SCOPES = [AI_FEATURES].freeze PROFILE_SCOPE = :profile EMAIL_SCOPE = :email @@ -429,7 +436,7 @@ module Gitlab end def non_admin_available_scopes - API_SCOPES + REPOSITORY_SCOPES + registry_scopes + OBSERVABILITY_SCOPES + API_SCOPES + REPOSITORY_SCOPES + registry_scopes + OBSERVABILITY_SCOPES + AI_FEATURES_SCOPES end def find_build_by_token(token) diff --git a/lib/gitlab/auth/devise/strategies/combined_two_factor_authenticatable.rb b/lib/gitlab/auth/devise/strategies/combined_two_factor_authenticatable.rb new file mode 100644 index 00000000000..ef326fd2a99 --- /dev/null +++ b/lib/gitlab/auth/devise/strategies/combined_two_factor_authenticatable.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Devise + module Strategies + # This strategy combines the following strategies from + # devise_two_factor gem: + # - TwoFactorAuthenticatable: https://github.com/devise-two-factor/devise-two-factor/blob/v4.0.2/lib/devise_two_factor/strategies/two_factor_authenticatable.rb + # - TwoFactorBackupable: https://github.com/devise-two-factor/devise-two-factor/blob/v4.0.2/lib/devise_two_factor/strategies/two_factor_backupable.rb + # to avoid double incrementing failed login attempts counter by each + # strategy in case an incorrect password is provided. + class CombinedTwoFactorAuthenticatable < ::Devise::Strategies::DatabaseAuthenticatable + def authenticate! + resource = mapping.to.find_for_database_authentication(authentication_hash) + + # We check the OTP / backup code, then defer to DatabaseAuthenticatable + is_valid = validate(resource) do + validate_otp(resource) || resource.invalidate_otp_backup_code!(params[scope]['otp_attempt']) + end + + if is_valid + # Devise fails to authenticate invalidated resources, but if we've + # gotten here, the object changed (Since we deleted a recovery code) + resource.save! + + super + end + + fail(::Devise.paranoid ? :invalid : :not_found_in_database) unless resource # rubocop: disable Style/SignalException + + # We want to cascade to the next strategy if this one fails, + # but database authenticatable automatically halts on a bad password + @halted = false if @result == :failure + end + + def validate_otp(resource) + return true unless resource.otp_required_for_login + + return if params[scope]['otp_attempt'].nil? + + resource.validate_and_consume_otp!(params[scope]['otp_attempt']) + end + end + end + end + end +end + +Warden::Strategies.add( + :combined_two_factor_authenticatable, + Gitlab::Auth::Devise::Strategies::CombinedTwoFactorAuthenticatable) diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index 0201f1f8725..a91d99d1b76 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -41,7 +41,7 @@ module Gitlab ldap_search(base: dn, filter: filter, scope: Net::LDAP::SearchScope_BaseObject, - attributes: %w{dn}).any? + attributes: %w[dn]).any? end def ldap_search(*args) diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 13ca4f01154..ed7caf84558 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -198,8 +198,8 @@ module Gitlab def default_attributes { - 'username' => %W(#{uid} uid sAMAccountName userid).uniq, - 'email' => %w(mail email userPrincipalName), + 'username' => %W[#{uid} uid sAMAccountName userid].uniq, + 'email' => %w[mail email userPrincipalName], 'name' => 'cn', 'first_name' => 'givenName', 'last_name' => 'sn' diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index 2ce8677c8b7..1f07baefa67 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -33,7 +33,7 @@ module Gitlab end def self.providers - Devise.omniauth_providers + ::Devise.omniauth_providers end def self.enabled?(name) @@ -69,7 +69,7 @@ module Gitlab end else provider = Gitlab.config.omniauth.providers.find do |provider| - provider.name == name || (provider.name == 'openid_connect' && provider.args.name == name) + provider.name == name || (provider.name == 'openid_connect' && provider.dig(:args, :name) == name) end merge_provider_args_with_defaults!(provider) diff --git a/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads.rb new file mode 100644 index 00000000000..7f0ee54012b --- /dev/null +++ b/lib/gitlab/background_migration/backfill_has_merge_request_of_vulnerability_reads.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Sets the `has_merge_request` of the existing `vulnerability_reads` records + class BackfillHasMergeRequestOfVulnerabilityReads < BatchedMigrationJob + operation_name :set_has_merge_request + feature_category :database + + UPDATE_SQL = <<~SQL + UPDATE + vulnerability_reads + SET + has_merge_request = true + FROM + (%<subquery>s) as sub_query + WHERE + vulnerability_reads.vulnerability_id = sub_query.vulnerability_id + SQL + + def perform + each_sub_batch do |sub_batch| + update_query = update_query_for(sub_batch) + + connection.execute(update_query) + end + end + + private + + def update_query_for(sub_batch) + subquery = sub_batch.joins(" + INNER JOIN vulnerability_merge_request_links ON + vulnerability_reads.vulnerability_id = + vulnerability_merge_request_links.vulnerability_id") + + format(UPDATE_SQL, subquery: subquery.to_sql) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb index b0b7882d54d..228269adc04 100644 --- a/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb +++ b/lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb @@ -14,7 +14,7 @@ module Gitlab batch_metrics.time_operation(:update_all) do ApplicationRecord.connection.execute <<~SQL - WITH route_and_ns(route_id, project_namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + WITH route_and_ns(route_id, project_namespace_id) AS MATERIALIZED ( #{sub_batch.to_sql} ) UPDATE routes diff --git a/lib/gitlab/background_migration/backfill_note_discussion_id.rb b/lib/gitlab/background_migration/backfill_note_discussion_id.rb index ce2698b3cb8..d5f12e73ff8 100644 --- a/lib/gitlab/background_migration/backfill_note_discussion_id.rb +++ b/lib/gitlab/background_migration/backfill_note_discussion_id.rb @@ -37,7 +37,7 @@ module Gitlab { discussion_id: note.generate_discussion_id } end - Gitlab::Database::BulkUpdate.execute(%i(discussion_id), mapping) + Gitlab::Database::BulkUpdate.execute(%i[discussion_id], mapping) end end end diff --git a/lib/gitlab/background_migration/backfill_nuget_normalized_version.rb b/lib/gitlab/background_migration/backfill_nuget_normalized_version.rb new file mode 100644 index 00000000000..26ca5f299b4 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_nuget_normalized_version.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill the normalized_version column in the packages_nuget_metadata table + class BackfillNugetNormalizedVersion < BatchedMigrationJob + operation_name :update_all + feature_category :package_registry + + scope_to ->(relation) { relation.where(normalized_version: nil) } + + # rubocop: disable Style/Documentation + class Package < ApplicationRecord + self.table_name = 'packages_packages' + end + + class PackagesNugetMetadatum < ApplicationRecord + self.table_name = 'packages_nuget_metadata' + + LEADING_ZEROES_REGEX = /^(?!0$)0+(?=\d)/ + + belongs_to :package + delegate :version, to: :package, prefix: true + + def set_normalized_version + return unless package + + self.normalized_version = normalize + end + + private + + def normalize + version = remove_leading_zeroes + version = remove_build_metadata(version) + version = omit_zero_in_fourth_part(version) + append_suffix(version) + end + + def remove_leading_zeroes + package_version.split('.').map { |part| part.sub(LEADING_ZEROES_REGEX, '') }.join('.') + end + + def remove_build_metadata(version) + version.split('+').first.downcase + end + + def omit_zero_in_fourth_part(version) + parts = version.split('.') + parts[3] = nil if parts.fourth == '0' && parts.third.exclude?('-') + parts.compact.join('.') + end + + def append_suffix(version) + version << '.0.0' if version.count('.') == 0 + version << '.0' if version.count('.') == 1 + version + end + end + # rubocop: enable Style/Documentation + + def perform + each_sub_batch do |sub_batch| + sub_batch.select(:package_id).then do |relation| + connection.execute(update_query(relation)) + end + end + end + + private + + def update_query(relation) + <<-SQL.squish + UPDATE packages_nuget_metadata + SET normalized_version = v.normalized_version + FROM ( VALUES #{package_id_and_normalized_version(relation)} ) AS v(package_id, normalized_version) + WHERE packages_nuget_metadata.package_id = v.package_id; + SQL + end + + def package_id_and_normalized_version(relation) + packages = Package + .where(id: relation.map(&:package_id)) + .select(:id, :version) + .index_by(&:id) + + # We need a new PackagesNugetMetadatum instance to be able to trigger + # #set_normalized_version method that sets the normalized_version. + relation.map do |record| + new_record = PackagesNugetMetadatum.new(record.attributes) + new_record.package = packages[record.package_id] + new_record.set_normalized_version + "(#{record.package_id}, '#{new_record.normalized_version}')" + end.join(', ') + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_statistics_storage_size_with_recent_size.rb b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_with_recent_size.rb new file mode 100644 index 00000000000..40daedffa64 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_statistics_storage_size_with_recent_size.rb @@ -0,0 +1,165 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class BackfillProjectStatisticsStorageSizeWithRecentSize < BatchedMigrationJob + RECENT_OBJECTS_SIZE_ENABLED_AT = Date.new(2023, 8, 8).freeze + + class Route < ::ApplicationRecord + belongs_to :source, inverse_of: :route, polymorphic: true + end + + class Namespace < ::ApplicationRecord + self.table_name = 'namespaces' + self.inheritance_column = nil # rubocop:disable Database/AvoidInheritanceColumn + + include Routable + + belongs_to :parent, class_name: 'Namespace', inverse_of: 'namespaces' + + has_one :route, -> { where(source_type: 'Namespace') }, inverse_of: :source, foreign_key: :source_id + + has_many :projects, inverse_of: :parent + has_many :namespaces, inverse_of: :parent + end + + class Project < ::ApplicationRecord + self.table_name = 'projects' + + include Routable + + has_one :statistics, class_name: '::Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithRecentSize::ProjectStatistics' # rubocop:disable Layout/LineLength + has_one :route, -> { where(source_type: 'Project') }, inverse_of: :source, foreign_key: :source_id + belongs_to :parent, class_name: 'Namespace', foreign_key: :namespace_id, inverse_of: 'projects' + + delegate :disk_path, to: :storage + + def repository + Gitlab::GlRepository::PROJECT.repository_for(self) + end + + def storage + Storage::Hashed.new(self) + end + end + + module Storage + class Hashed + attr_accessor :project + + ROOT_PATH_PREFIX = '@hashed' + + def initialize(project) + @project = project + end + + def disk_path + "#{ROOT_PATH_PREFIX}/#{disk_hash[0..1]}/#{disk_hash[2..3]}/#{disk_hash}" + end + + def disk_hash + @disk_hash ||= Digest::SHA2.hexdigest(project.id.to_s) + end + end + end + + class ProjectStatistics < ::ApplicationRecord + include ::EachBatch + + self.table_name = 'project_statistics' + + belongs_to :project, class_name: '::Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithRecentSize::Project' # rubocop:disable Layout/LineLength + + def update_repository_size + self.repository_size = project.repository.recent_objects_size.megabytes + end + + def update_storage_size(storage_size_components) + return unless repository_size > 0 + + update_repository_size + + new_storage_size = storage_size_components.sum { |component| method(component).call } + + # Only update storage_size if storage_size needs updating + return unless storage_size != new_storage_size + + self.storage_size = new_storage_size + save! + + ::Namespaces::ScheduleAggregationWorker.perform_async(project.namespace_id) + log_with_data('Scheduled Namespaces::ScheduleAggregationWorker') + end + + private + + def log_with_data(log_line) + log_info( + log_line, + project_id: project.id, + pipeline_artifacts_size: pipeline_artifacts_size, + storage_size: storage_size, + namespace_id: project.namespace_id + ) + end + + def log_info(message, **extra) + ::Gitlab::BackgroundMigration::Logger.info( + migrator: 'BackfillProjectStatisticsStorageSizeWithRecentSize', + message: message, + **extra + ) + end + + def wiki_size + super.to_i + end + + def snippets_size + super.to_i + end + end + + scope_to ->(relation) do + scope = relation.where('repository_size > 0') + + if Gitlab.dev_or_test_env? || Gitlab.org_or_com? + scope = scope.where('updated_at < ?', RECENT_OBJECTS_SIZE_ENABLED_AT) + end + + scope + end + + operation_name :update_storage_size + + feature_category :consumables_cost_management + + def perform + each_sub_batch do |sub_batch| + ProjectStatistics.merge(sub_batch).each do |statistics| + statistics.update_storage_size(storage_size_components) + end + end + end + + private + + # Overridden in EE + def storage_size_components + [ + :repository_size, + :wiki_size, + :lfs_objects_size, + :build_artifacts_size, + :packages_size, + :snippets_size, + :uploads_size + ] + end + end + end + # rubocop:enable Style/Documentation +end + +Gitlab::BackgroundMigration::BackfillProjectStatisticsStorageSizeWithRecentSize.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_snippet_repositories.rb b/lib/gitlab/background_migration/backfill_snippet_repositories.rb index b58f0a3a3e0..15ea4ab26e9 100644 --- a/lib/gitlab/background_migration/backfill_snippet_repositories.rb +++ b/lib/gitlab/background_migration/backfill_snippet_repositories.rb @@ -129,7 +129,7 @@ module Gitlab end def migration_bot_user - @migration_bot_user ||= User.migration_bot + @migration_bot_user ||= Users::Internal.migration_bot end # We sometimes receive invalid path errors from Gitaly if the Snippet filename diff --git a/lib/gitlab/background_migration/backfill_user_preferences_with_defaults.rb b/lib/gitlab/background_migration/backfill_user_preferences_with_defaults.rb new file mode 100644 index 00000000000..2fba6e66d48 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_user_preferences_with_defaults.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the user_preferences table columns with their default values + class BackfillUserPreferencesWithDefaults < BatchedMigrationJob + operation_name :backfill_user_preferences_with_defaults + feature_category :user_profile + + def perform + each_sub_batch do |sub_batch| + connection.transaction do + sub_batch.where(tab_width: nil).update_all(tab_width: 8) + sub_batch.where(time_display_relative: nil).update_all(time_display_relative: true) + sub_batch.where(render_whitespace_in_code: nil).update_all(render_whitespace_in_code: false) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_users_with_defaults.rb b/lib/gitlab/background_migration/backfill_users_with_defaults.rb new file mode 100644 index 00000000000..10b4963163d --- /dev/null +++ b/lib/gitlab/background_migration/backfill_users_with_defaults.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the users table columns with their default values + class BackfillUsersWithDefaults < BatchedMigrationJob + operation_name :backfill_users_with_defaults + feature_category :user_profile + + def perform + each_sub_batch do |sub_batch| + connection.transaction do + sub_batch.where(project_view: nil).update_all(project_view: 2) + sub_batch.where(hide_no_ssh_key: nil).update_all(hide_no_ssh_key: false) + sub_batch.where(hide_no_password: nil).update_all(hide_no_password: false) + sub_batch.where(notified_of_own_activity: nil).update_all(notified_of_own_activity: false) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_workspace_personal_access_token.rb b/lib/gitlab/background_migration/backfill_workspace_personal_access_token.rb new file mode 100644 index 00000000000..f71759dc8dd --- /dev/null +++ b/lib/gitlab/background_migration/backfill_workspace_personal_access_token.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # No op on ce + class BackfillWorkspacePersonalAccessToken < BatchedMigrationJob + feature_category :remote_development + def perform; end + end + end +end + +Gitlab::BackgroundMigration::BackfillWorkspacePersonalAccessToken.prepend_mod_with('Gitlab::BackgroundMigration::BackfillWorkspacePersonalAccessToken') # rubocop:disable Layout/LineLength diff --git a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb index c221f8ea411..d6695eddf02 100644 --- a/lib/gitlab/background_migration/cleanup_orphaned_routes.rb +++ b/lib/gitlab/background_migration/cleanup_orphaned_routes.rb @@ -40,7 +40,7 @@ module Gitlab non_orphaned_namespace_routes.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| batch_metrics.time_operation(:fix_missing_namespace_id) do ApplicationRecord.connection.execute <<~SQL - WITH route_and_ns(route_id, namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + WITH route_and_ns(route_id, namespace_id) AS MATERIALIZED ( #{sub_batch.to_sql} ) UPDATE routes diff --git a/lib/gitlab/background_migration/convert_credit_card_validation_data_to_hashes.rb b/lib/gitlab/background_migration/convert_credit_card_validation_data_to_hashes.rb new file mode 100644 index 00000000000..7bb3b661caa --- /dev/null +++ b/lib/gitlab/background_migration/convert_credit_card_validation_data_to_hashes.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Converts a credit card's expiration_date, last_digits, network & holder_name + # to hash and store values in new columns + class ConvertCreditCardValidationDataToHashes < BatchedMigrationJob + operation_name :convert_credit_card_data + feature_category :user_profile + + class CreditCardValidation < ApplicationRecord # rubocop:disable Style/Documentation + self.table_name = 'user_credit_card_validations' + end + + def perform + each_sub_batch do |sub_batch| + credit_cards = CreditCardValidation.where(user_id: sub_batch) + + credit_card_hashes = credit_cards.map do |c| + { + user_id: c.user_id, + credit_card_validated_at: c.credit_card_validated_at, + last_digits_hash: hashed_value(c.last_digits), + holder_name_hash: hashed_value(c.holder_name&.downcase), + network_hash: hashed_value(c.network&.downcase), + expiration_date_hash: hashed_value(c.expiration_date&.to_s) + } + end + + CreditCardValidation.upsert_all(credit_card_hashes) + end + end + + def hashed_value(value) + Gitlab::CryptoHelper.sha256(value) if value.present? + end + end + end +end diff --git a/lib/gitlab/background_migration/create_compliance_standards_adherence.rb b/lib/gitlab/background_migration/create_compliance_standards_adherence.rb new file mode 100644 index 00000000000..a79dc107cc7 --- /dev/null +++ b/lib/gitlab/background_migration/create_compliance_standards_adherence.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class CreateComplianceStandardsAdherence < BatchedMigrationJob + feature_category :compliance_management + + def perform + # no-op. The logic is defined in EE module. + end + end + # rubocop: enable Style/Documentation + end +end + +::Gitlab::BackgroundMigration::CreateComplianceStandardsAdherence.prepend_mod diff --git a/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb b/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb index 6de2187b8e3..5225fb6aced 100644 --- a/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb +++ b/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb @@ -41,7 +41,7 @@ module Gitlab .where('issue_metrics.first_mentioned_in_commit_at > first_authored_date.authored_date') TmpIssueMetrics.connection.execute <<~UPDATE_METRICS - WITH cte AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + WITH cte AS MATERIALIZED ( #{inner_query.to_sql} ) UPDATE issue_metrics diff --git a/lib/gitlab/background_migration/fix_namespace_ids_of_vulnerability_reads.rb b/lib/gitlab/background_migration/fix_namespace_ids_of_vulnerability_reads.rb new file mode 100644 index 00000000000..ffa1d0f4157 --- /dev/null +++ b/lib/gitlab/background_migration/fix_namespace_ids_of_vulnerability_reads.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class FixNamespaceIdsOfVulnerabilityReads < BatchedMigrationJob + feature_category :vulnerability_management + + def perform + # no-op for the FOSS version. + end + end + # rubocop: enable Style/Documentation + end +end + +::Gitlab::BackgroundMigration::FixNamespaceIdsOfVulnerabilityReads.prepend_mod diff --git a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb index c21f9c1d50f..45a93f158d8 100644 --- a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb +++ b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb @@ -22,7 +22,7 @@ module Gitlab def sql(from_id, to_id) <<~SQL - WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + WITH created_records AS MATERIALIZED ( INSERT INTO project_features ( project_id, merge_requests_access_level, diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb index 452167d4d61..41326060a05 100644 --- a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb +++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb @@ -136,7 +136,7 @@ module Gitlab # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT def create_sql(from_id, to_id) <<~SQL - WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + WITH created_records AS MATERIALIZED ( INSERT INTO services (project_id, #{DEFAULTS.keys.map { |key| %("#{key}") }.join(',')}, created_at, updated_at) #{select_insert_values_sql(from_id, to_id)} RETURNING * @@ -149,7 +149,7 @@ module Gitlab # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT def update_sql(from_id, to_id) <<~SQL - WITH updated_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + WITH updated_records AS MATERIALIZED ( UPDATE services SET active = TRUE WHERE services.project_id BETWEEN #{Integer(from_id)} AND #{Integer(to_id)} AND services.properties = '{}' AND services.type = '#{Migratable::PrometheusService.type}' AND #{group_cluster_condition(from_id, to_id)} AND services.active = FALSE diff --git a/lib/gitlab/background_migration/populate_denormalized_columns_for_sbom_occurrences.rb b/lib/gitlab/background_migration/populate_denormalized_columns_for_sbom_occurrences.rb new file mode 100644 index 00000000000..0996db01f8a --- /dev/null +++ b/lib/gitlab/background_migration/populate_denormalized_columns_for_sbom_occurrences.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class PopulateDenormalizedColumnsForSbomOccurrences < BatchedMigrationJob + feature_category :dependency_management + + def perform + # no-op for the FOSS version + end + end + # rubocop:enable Style/Documentation + end +end + +::Gitlab::BackgroundMigration::PopulateDenormalizedColumnsForSbomOccurrences.prepend_mod diff --git a/lib/gitlab/background_migration/populate_projects_star_count.rb b/lib/gitlab/background_migration/populate_projects_star_count.rb index 0790bd98018..97e2d9b81ac 100644 --- a/lib/gitlab/background_migration/populate_projects_star_count.rb +++ b/lib/gitlab/background_migration/populate_projects_star_count.rb @@ -42,7 +42,7 @@ module Gitlab ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/421843') do ApplicationRecord.connection.execute <<~SQL - WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{sub_batch.select(:id).to_sql}) + WITH batched_relation AS MATERIALIZED (#{sub_batch.select(:id).to_sql}) UPDATE projects SET star_count = ( SELECT COUNT(*) diff --git a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb index 845a3c16bbe..e21d7951f59 100644 --- a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb +++ b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb @@ -90,7 +90,7 @@ module Gitlab .select("namespaces.id, namespaces.tmp_project_id") ApplicationRecord.connection.execute <<~SQL - WITH cte(project_namespace_id, project_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + WITH cte(project_namespace_id, project_id) AS MATERIALIZED ( #{projects.to_sql} ) UPDATE projects diff --git a/lib/gitlab/background_migration/rebalance_partition_id.rb b/lib/gitlab/background_migration/rebalance_partition_id.rb deleted file mode 100644 index 7000ae5a856..00000000000 --- a/lib/gitlab/background_migration/rebalance_partition_id.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This rebalances partition_id to fix invalid records in production - class RebalancePartitionId < BatchedMigrationJob - INVALID_PARTITION_ID = 101 - VALID_PARTITION_ID = 100 - - scope_to ->(relation) { relation.where(partition_id: INVALID_PARTITION_ID) } - operation_name :update_all - feature_category :continuous_integration - - def perform - each_sub_batch do |sub_batch| - sub_batch.update_all(partition_id: VALID_PARTITION_ID) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/sync_scan_result_policies.rb b/lib/gitlab/background_migration/sync_scan_result_policies.rb new file mode 100644 index 00000000000..0b68e4312f3 --- /dev/null +++ b/lib/gitlab/background_migration/sync_scan_result_policies.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration to sync scan result policies from YAML to DB table by kicking off sync Sidekiq jobs + class SyncScanResultPolicies < BatchedMigrationJob + feature_category :security_policy_management + + def perform; end + end + end +end + +Gitlab::BackgroundMigration::SyncScanResultPolicies.prepend_mod diff --git a/lib/gitlab/background_migration/update_users_set_external_if_service_account.rb b/lib/gitlab/background_migration/update_users_set_external_if_service_account.rb new file mode 100644 index 00000000000..c8f1847aca3 --- /dev/null +++ b/lib/gitlab/background_migration/update_users_set_external_if_service_account.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class is responsible for updating users external field if service account. + class UpdateUsersSetExternalIfServiceAccount < BatchedMigrationJob + operation_name :update # This is used as the key on collecting metrics + scope_to ->(relation) { relation.where(user_type: HasUserType::USER_TYPES[:service_account]) } + feature_category :system_access + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(external: true) + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index e785ce558db..7f228c19b6b 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -210,7 +210,11 @@ module Gitlab source_branch_sha = pull_request.source_branch_sha target_branch_sha = pull_request.target_branch_sha - source_branch_sha = project.repository.commit(source_branch_sha)&.sha || source_branch_sha + + source_sha_from_commit_sha = project.repository.commit(source_branch_sha)&.sha + source_sha_from_merge_sha = project.repository.commit(pull_request.merge_commit_sha)&.sha + + source_branch_sha = source_sha_from_commit_sha || source_sha_from_merge_sha || source_branch_sha target_branch_sha = project.repository.commit(target_branch_sha)&.sha || target_branch_sha merge_request = project.merge_requests.create!( diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb new file mode 100644 index 00000000000..d76e08e1039 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class PullRequestImporter + include Loggable + + def initialize(project, hash) + @project = project + @formatter = Gitlab::ImportFormatter.new + @user_finder = UserFinder.new(project) + @object = hash.with_indifferent_access + end + + def execute + log_info(import_stage: 'import_pull_request', message: 'starting', iid: object[:iid]) + + description = '' + description += author_line + description += object[:description] if object[:description] + + attributes = { + iid: object[:iid], + title: object[:title], + description: description, + source_project_id: project.id, + source_branch: Gitlab::Git.ref_name(object[:source_branch_name]), + source_branch_sha: source_branch_sha, + target_project_id: project.id, + target_branch: Gitlab::Git.ref_name(object[:target_branch_name]), + target_branch_sha: object[:target_branch_sha], + state_id: MergeRequest.available_states[object[:state]], + author_id: author_id, + created_at: object[:created_at], + updated_at: object[:updated_at] + } + + creator = Gitlab::Import::MergeRequestCreator.new(project) + + merge_request = creator.execute(attributes) + + if merge_request + merge_request.assignee_ids = [author_id] + merge_request.reviewer_ids = reviewers + merge_request.save! + end + + log_info(import_stage: 'import_pull_request', message: 'finished', iid: object[:iid]) + rescue StandardError => e + Gitlab::Import::ImportFailureService.track(project_id: project.id, exception: e) + end + + private + + attr_reader :object, :project, :formatter, :user_finder + + def author_line + return '' if find_user_id + + formatter.author_line(object[:author]) + end + + def find_user_id + user_finder.find_user_id(object[:author]) + end + + def author_id + user_finder.gitlab_user_id(project, object[:author]) + end + + def reviewers + return [] unless object[:reviewers].present? + + object[:reviewers].filter_map do |reviewer| + user_finder.find_user_id(reviewer) + end + end + + def source_branch_sha + project.repository.commit(object[:source_branch_sha])&.sha || + project.repository.commit(object[:merge_commit_sha])&.sha || + object[:source_branch_sha] + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb new file mode 100644 index 00000000000..1c7ce7f2f3a --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class PullRequestsImporter + include ParallelScheduling + + def execute + log_info(import_stage: 'import_pull_requests', message: 'importing pull requests') + + pull_requests = client.pull_requests(project.import_source) + + pull_requests.each do |pull_request| + job_waiter.jobs_remaining += 1 + + next if already_enqueued?(pull_request) + + job_delay = calculate_job_delay(job_waiter.jobs_remaining) + + sidekiq_worker_class.perform_in(job_delay, project.id, pull_request.to_hash, job_waiter.key) + + mark_as_enqueued(pull_request) + end + + job_waiter + rescue StandardError => e + track_import_failure!(project, exception: e) + end + + private + + def sidekiq_worker_class + ImportPullRequestWorker + end + + def collection_method + :pull_requests + end + + def id_for_already_enqueued_cache(object) + object.iid + end + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_import/importers/repository_importer.rb new file mode 100644 index 00000000000..7b0362b6ec6 --- /dev/null +++ b/lib/gitlab/bitbucket_import/importers/repository_importer.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Importers + class RepositoryImporter + include Loggable + + def initialize(project) + @project = project + end + + def execute + log_info(import_stage: 'import_repository', message: 'starting import') + + if project.empty_repo? + project.repository.import_repository(project.import_url) + project.repository.fetch_as_mirror(project.import_url, refmap: refmap) + + validate_repository_size! + + update_clone_time + end + + import_wiki + + log_info(import_stage: 'import_repository', message: 'finished import') + + true + rescue ::Gitlab::Git::CommandError => e + Gitlab::ErrorTracking.log_exception( + e, import_stage: 'import_repository', message: 'failed import', error: e.message + ) + + # Expire cache to prevent scenarios such as: + # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true + # 2. Retried import, repo is broken or not imported but +exists?+ still returns true + project.repository.expire_content_cache if project.repository_exists? + + raise + end + + private + + attr_reader :project + + def refmap + # We omit :heads and :tags since these are fetched in the import_repository + ['+refs/pull-requests/*/to:refs/merge-requests/*/head'] + end + + def import_wiki + return if project.wiki.repository_exists? + + project.wiki.repository.import_repository(wiki.import_url) + rescue StandardError => e + Gitlab::ErrorTracking.log_exception( + e, import_stage: 'import_repository', message: 'failed to import wiki', error: e.message + ) + end + + def wiki + WikiFormatter.new(project) + end + + def update_clone_time + project.touch(:last_repository_updated_at) + end + + def validate_repository_size! + # Defined in EE + end + end + end + end +end + +Gitlab::BitbucketImport::Importers::RepositoryImporter.prepend_mod diff --git a/lib/gitlab/bitbucket_import/loggable.rb b/lib/gitlab/bitbucket_import/loggable.rb new file mode 100644 index 00000000000..eda3cc96d4d --- /dev/null +++ b/lib/gitlab/bitbucket_import/loggable.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module Loggable + def log_debug(messages) + logger.debug(log_data(messages)) + end + + def log_info(messages) + logger.info(log_data(messages)) + end + + def log_warn(messages) + logger.warn(log_data(messages)) + end + + def log_error(messages) + logger.error(log_data(messages)) + end + + private + + def logger + Gitlab::BitbucketImport::Logger + end + + def log_data(messages) + messages.merge(log_base_data) + end + + def log_base_data + { + class: self.class.name, + project_id: project.id, + project_path: project.full_path + } + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/logger.rb b/lib/gitlab/bitbucket_import/logger.rb new file mode 100644 index 00000000000..1f4d175f8a3 --- /dev/null +++ b/lib/gitlab/bitbucket_import/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + class Logger < ::Gitlab::Import::Logger + def default_attributes + super.merge(import_type: :bitbucket) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/parallel_importer.rb b/lib/gitlab/bitbucket_import/parallel_importer.rb new file mode 100644 index 00000000000..1563261fa4a --- /dev/null +++ b/lib/gitlab/bitbucket_import/parallel_importer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + class ParallelImporter + def self.async? + true + end + + def self.imports_repository? + true + end + + def self.track_start_import(project) + Gitlab::Import::Metrics.new(:bitbucket_importer, project).track_start_import + end + + def initialize(project) + @project = project + end + + def execute + Gitlab::Import::SetAsyncJid.set_jid(project.import_state) + + Stage::ImportRepositoryWorker + .with_status + .perform_async(project.id) + + true + end + + private + + attr_reader :project + end + end +end diff --git a/lib/gitlab/bitbucket_import/parallel_scheduling.rb b/lib/gitlab/bitbucket_import/parallel_scheduling.rb new file mode 100644 index 00000000000..f4df9a35526 --- /dev/null +++ b/lib/gitlab/bitbucket_import/parallel_scheduling.rb @@ -0,0 +1,93 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + module ParallelScheduling + include Loggable + + attr_reader :project, :already_enqueued_cache_key, :job_waiter_cache_key + + # The base cache key to use for tracking already enqueued objects. + ALREADY_ENQUEUED_CACHE_KEY = + 'bitbucket-importer/already-enqueued/%{project}/%{collection}' + + # The base cache key to use for storing job waiter key + JOB_WAITER_CACHE_KEY = + 'bitbucket-importer/job-waiter/%{project}/%{collection}' + + BATCH_SIZE = 100 + + # project - An instance of `Project`. + def initialize(project) + @project = project + + @already_enqueued_cache_key = + format(ALREADY_ENQUEUED_CACHE_KEY, project: project.id, collection: collection_method) + @job_waiter_cache_key = + format(JOB_WAITER_CACHE_KEY, project: project.id, collection: collection_method) + end + + private + + def client + @client ||= Bitbucket::Client.new(project.import_data.credentials) + end + + # Returns the ID to use for the cache used for checking if an object has + # already been enqueued or not. + # + # object - The object we may want to import. + def id_for_already_enqueued_cache(object) + raise NotImplementedError + end + + # The Sidekiq worker class used for scheduling the importing of objects in + # parallel. + def sidekiq_worker_class + raise NotImplementedError + end + + # The name of the method to call to retrieve the data to import. + def collection_method + raise NotImplementedError + end + + def job_waiter + @job_waiter ||= begin + key = Gitlab::Cache::Import::Caching.read(job_waiter_cache_key) + key ||= Gitlab::Cache::Import::Caching.write(job_waiter_cache_key, JobWaiter.generate_key) + + JobWaiter.new(0, key) + end + end + + def already_enqueued?(object) + id = id_for_already_enqueued_cache(object) + + Gitlab::Cache::Import::Caching.set_includes?(already_enqueued_cache_key, id) + end + + # Marks the given object as "already enqueued". + def mark_as_enqueued(object) + id = id_for_already_enqueued_cache(object) + + Gitlab::Cache::Import::Caching.set_add(already_enqueued_cache_key, id) + end + + def calculate_job_delay(job_index) + multiplier = (job_index / BATCH_SIZE) + + (multiplier * 1.minute) + 1.second + end + + def track_import_failure!(project, exception:, **args) + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: self.class.name, + exception: exception, + **args + ) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/user_finder.rb b/lib/gitlab/bitbucket_import/user_finder.rb new file mode 100644 index 00000000000..70ed94351d5 --- /dev/null +++ b/lib/gitlab/bitbucket_import/user_finder.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module BitbucketImport + class UserFinder + USER_ID_FOR_AUTHOR_CACHE_KEY = 'bitbucket-importer/user-finder/%{project_id}/%{author}' + CACHE_USER_ID_NOT_FOUND = -1 + + attr_reader :project + + def initialize(project) + @project = project + end + + def find_user_id(author) + return unless author + + cache_key = build_cache_key(author) + cached_id = cache.read_integer(cache_key) + + return if cached_id == CACHE_USER_ID_NOT_FOUND + return cached_id if cached_id + + id = User.by_provider_and_extern_uid(:bitbucket, author).select(:id).first&.id + + cache.write(cache_key, id || CACHE_USER_ID_NOT_FOUND) + + id + end + + def gitlab_user_id(project, username) + find_user_id(username) || project.creator_id + end + + private + + def cache + Cache::Import::Caching + end + + def build_cache_key(author) + format(USER_ID_FOR_AUTHOR_CACHE_KEY, project_id: project.id, author: author) + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb deleted file mode 100644 index 1871dd3a89d..00000000000 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ /dev/null @@ -1,468 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BitbucketServerImport - class Importer - include Loggable - - attr_reader :recover_missing_commits - attr_reader :project, :project_key, :repository_slug, :client, :errors, :users, :already_imported_cache_key - - BATCH_SIZE = 100 - # The base cache key to use for tracking already imported objects. - ALREADY_IMPORTED_CACHE_KEY = - 'bitbucket_server-importer/already-imported/%{project}/%{collection}' - - TempBranch = Struct.new(:name, :sha) - - def self.imports_repository? - true - end - - def self.refmap - # We omit :heads and :tags since these are fetched in the import_repository - ['+refs/pull-requests/*/to:refs/merge-requests/*/head'] - end - - # Unlike GitHub, you can't grab the commit SHAs for pull requests that - # have been closed but not merged even though Bitbucket has these - # commits internally. We can recover these pull requests by creating a - # branch with the Bitbucket REST API, but by default we turn this - # behavior off. - def initialize(project, recover_missing_commits: false) - @project = project - @recover_missing_commits = recover_missing_commits - @project_key = project.import_data.data['project_key'] - @repository_slug = project.import_data.data['repo_slug'] - @client = BitbucketServer::Client.new(project.import_data.credentials) - @formatter = Gitlab::ImportFormatter.new - @errors = [] - @users = {} - @temp_branches = [] - @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % - { project: project.id, collection: collection_method } - end - - def collection_method - :pull_requests - end - - def execute - import_repository - import_pull_requests - download_lfs_objects - delete_temp_branches - handle_errors - metrics.track_finished_import - - log_info(import_stage: "complete") - - Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT) - true - end - - private - - def handle_errors - return unless errors.any? - - project.import_state.update_column(:last_error, { - message: 'The remote data could not be fully imported.', - errors: errors - }.to_json) - end - - def find_user_id(by:, value:) - return unless value - - return users[value] if users.key?(value) - - user = if by == :email - User.find_by_any_email(value, confirmed: true) - else - User.find_by_username(value) - end - - users[value] = user&.id - - user&.id - end - - def repo - @repo ||= client.repo(project_key, repository_slug) - end - - def sha_exists?(sha) - project.repository.commit(sha) - end - - def temp_branch_name(pull_request, suffix) - "gitlab/import/pull-request/#{pull_request.iid}/#{suffix}" - end - - # This method restores required SHAs that GitLab needs to create diffs - # into branch names as the following: - # - # gitlab/import/pull-request/N/{to,from} - def restore_branches(pull_requests) - shas_to_restore = [] - - pull_requests.each do |pull_request| - shas_to_restore << TempBranch.new(temp_branch_name(pull_request, :from), - pull_request.source_branch_sha) - shas_to_restore << TempBranch.new(temp_branch_name(pull_request, :to), - pull_request.target_branch_sha) - end - - # Create the branches on the Bitbucket Server first - created_branches = restore_branch_shas(shas_to_restore) - - @temp_branches += created_branches - # Now sync the repository so we get the new branches - import_repository unless created_branches.empty? - end - - def restore_branch_shas(shas_to_restore) - shas_to_restore.each_with_object([]) do |temp_branch, branches_created| - branch_name = temp_branch.name - sha = temp_branch.sha - - next if sha_exists?(sha) - - begin - client.create_branch(project_key, repository_slug, branch_name, sha) - branches_created << temp_branch - rescue BitbucketServer::Connection::ConnectionError => e - log_warn(message: "Unable to recreate branch", sha: sha, error: e.message) - end - end - end - - def import_repository - log_info(import_stage: 'import_repository', message: 'starting import') - - project.repository.import_repository(project.import_url) - project.repository.fetch_as_mirror(project.import_url, refmap: self.class.refmap) - - log_info(import_stage: 'import_repository', message: 'finished import') - rescue ::Gitlab::Git::CommandError => e - Gitlab::ErrorTracking.log_exception( - e, - import_stage: 'import_repository', message: 'failed import', error: e.message - ) - - # Expire cache to prevent scenarios such as: - # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true - # 2. Retried import, repo is broken or not imported but +exists?+ still returns true - project.repository.expire_content_cache if project.repository_exists? - - raise - end - - def download_lfs_objects - result = Projects::LfsPointers::LfsImportService.new(project).execute - - if result[:status] == :error - errors << { type: :lfs_objects, errors: "The Lfs import process failed. #{result[:message]}" } - end - end - - # Bitbucket Server keeps tracks of references for open pull requests in - # refs/heads/pull-requests, but closed and merged requests get moved - # into hidden internal refs under stash-refs/pull-requests. Unless the - # SHAs involved are at the tip of a branch or tag, there is no way to - # retrieve the server for those commits. - # - # To avoid losing history, we use the Bitbucket API to re-create the branch - # on the remote server. Then we have to issue a `git fetch` to download these - # branches. - def import_pull_requests - page = 0 - - log_info(import_stage: 'import_pull_requests', message: "starting") - - loop do - log_debug(import_stage: 'import_pull_requests', message: "importing page #{page} and batch-size #{BATCH_SIZE} from #{page * BATCH_SIZE} to #{(page + 1) * BATCH_SIZE}") - - pull_requests = client.pull_requests(project_key, repository_slug, page_offset: page, limit: BATCH_SIZE).to_a - - break if pull_requests.empty? - - # Creating branches on the server and fetching the newly-created branches - # may take a number of network round-trips. This used to be done in batches to - # avoid doing a git fetch for every new branch, as the whole process is now - # batched, we do not need to separately do this in batches. - restore_branches(pull_requests) if recover_missing_commits - - pull_requests.each do |pull_request| - if already_imported?(pull_request) - log_info(import_stage: 'import_pull_requests', message: 'already imported', iid: pull_request.iid) - else - import_bitbucket_pull_request(pull_request) - end - rescue StandardError => e - Gitlab::ErrorTracking.log_exception( - e, - import_stage: 'import_pull_requests', iid: pull_request.iid, error: e.message - ) - - backtrace = Gitlab::BacktraceCleaner.clean_backtrace(e.backtrace) - errors << { type: :pull_request, iid: pull_request.iid, errors: e.message, backtrace: backtrace.join("\n"), raw_response: pull_request.raw } - end - - log_debug(import_stage: 'import_pull_requests', message: "finished page #{page} and batch-size #{BATCH_SIZE}") - page += 1 - end - end - - # Returns true if the given object has already been imported, false - # otherwise. - # - # object - The object to check. - def already_imported?(pull_request) - Gitlab::Cache::Import::Caching.set_includes?(already_imported_cache_key, pull_request.iid) - end - - # Marks the given object as "already imported". - def mark_as_imported(pull_request) - Gitlab::Cache::Import::Caching.set_add(already_imported_cache_key, pull_request.iid) - end - - def delete_temp_branches - @temp_branches.each do |branch| - client.delete_branch(project_key, repository_slug, branch.name, branch.sha) - project.repository.delete_branch(branch.name) - rescue BitbucketServer::Connection::ConnectionError => e - Gitlab::ErrorTracking.log_exception( - e, - import_stage: 'delete_temp_branches', branch: branch.name, error: e.message - ) - - @errors << { type: :delete_temp_branches, branch_name: branch.name, errors: e.message } - end - end - - def import_bitbucket_pull_request(pull_request) - log_info(import_stage: 'import_bitbucket_pull_requests', message: 'starting', iid: pull_request.iid) - - description = '' - description += author_line(pull_request) - description += pull_request.description if pull_request.description - - attributes = { - iid: pull_request.iid, - title: pull_request.title, - description: description, - reviewer_ids: reviewers(pull_request.reviewers), - source_project_id: project.id, - source_branch: Gitlab::Git.ref_name(pull_request.source_branch_name), - source_branch_sha: pull_request.source_branch_sha, - target_project_id: project.id, - target_branch: Gitlab::Git.ref_name(pull_request.target_branch_name), - target_branch_sha: pull_request.target_branch_sha, - state_id: MergeRequest.available_states[pull_request.state], - author_id: author_id(pull_request), - created_at: pull_request.created_at, - updated_at: pull_request.updated_at - } - - creator = Gitlab::Import::MergeRequestCreator.new(project) - merge_request = creator.execute(attributes) - - if merge_request.persisted? - import_pull_request_comments(pull_request, merge_request) - - metrics.merge_requests_counter.increment - end - - log_info(import_stage: 'import_bitbucket_pull_requests', message: 'finished', iid: pull_request.iid) - mark_as_imported(pull_request) - end - - def import_pull_request_comments(pull_request, merge_request) - log_info(import_stage: 'import_pull_request_comments', message: 'starting', iid: merge_request.iid) - - comments, other_activities = client.activities(project_key, repository_slug, pull_request.iid).partition(&:comment?) - - merge_event = other_activities.find(&:merge_event?) - import_merge_event(merge_request, merge_event) if merge_event - - inline_comments, pr_comments = comments.partition(&:inline_comment?) - - import_inline_comments(inline_comments.map(&:comment), merge_request) - import_standalone_pr_comments(pr_comments.map(&:comment), merge_request) - - log_info(import_stage: 'import_pull_request_comments', message: 'finished', iid: merge_request.iid, - merge_event_found: merge_event.present?, - inline_comments_count: inline_comments.count, - standalone_pr_comments: pr_comments.count) - end - - # rubocop: disable CodeReuse/ActiveRecord - def import_merge_event(merge_request, merge_event) - log_info(import_stage: 'import_merge_event', message: 'starting', iid: merge_request.iid) - - committer = merge_event.committer_email - - user_id = find_user_id(by: :email, value: committer) || project.creator_id - timestamp = merge_event.merge_timestamp - merge_request.update({ merge_commit_sha: merge_event.merge_commit }) - metric = MergeRequest::Metrics.find_or_initialize_by(merge_request: merge_request) - metric.update(merged_by_id: user_id, merged_at: timestamp) - - log_info(import_stage: 'import_merge_event', message: 'finished', iid: merge_request.iid) - end - # rubocop: enable CodeReuse/ActiveRecord - - def import_inline_comments(inline_comments, merge_request) - log_info(import_stage: 'import_inline_comments', message: 'starting', iid: merge_request.iid) - - inline_comments.each do |comment| - position = build_position(merge_request, comment) - parent = create_diff_note(merge_request, comment, position) - - next unless parent&.persisted? - - discussion_id = parent.discussion_id - - comment.comments.each do |reply| - create_diff_note(merge_request, reply, position, discussion_id) - end - end - - log_info(import_stage: 'import_inline_comments', message: 'finished', iid: merge_request.iid) - end - - def create_diff_note(merge_request, comment, position, discussion_id = nil) - attributes = pull_request_comment_attributes(comment) - attributes.merge!(position: position, type: 'DiffNote') - attributes[:discussion_id] = discussion_id if discussion_id - - note = merge_request.notes.build(attributes) - - if note.valid? - note.save - return note - end - - log_info(import_stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid) - - # Bitbucket Server supports the ability to comment on any line, not just the - # line in the diff. If we can't add the note as a DiffNote, fallback to creating - # a regular note. - create_fallback_diff_note(merge_request, comment, position) - rescue StandardError => e - Gitlab::ErrorTracking.log_exception( - e, - import_stage: 'create_diff_note', comment_id: comment.id, error: e.message - ) - - errors << { type: :pull_request, id: comment.id, errors: e.message } - nil - end - - def create_fallback_diff_note(merge_request, comment, position) - attributes = pull_request_comment_attributes(comment) - note = "*Comment on" - - note += " #{position.old_path}:#{position.old_line} -->" if position.old_line - note += " #{position.new_path}:#{position.new_line}" if position.new_line - note += "*\n\n#{comment.note}" - - attributes[:note] = note - merge_request.notes.create!(attributes) - end - - def build_position(merge_request, pr_comment) - params = { - diff_refs: merge_request.diff_refs, - old_path: pr_comment.file_path, - new_path: pr_comment.file_path, - old_line: pr_comment.old_pos, - new_line: pr_comment.new_pos - } - - Gitlab::Diff::Position.new(params) - end - - def import_standalone_pr_comments(pr_comments, merge_request) - pr_comments.each do |comment| - merge_request.notes.create!(pull_request_comment_attributes(comment)) - - comment.comments.each do |replies| - merge_request.notes.create!(pull_request_comment_attributes(replies)) - end - rescue StandardError => e - Gitlab::ErrorTracking.log_exception( - e, - import_stage: 'import_standalone_pr_comments', merge_request_id: merge_request.id, comment_id: comment.id, error: e.message - ) - - errors << { type: :pull_request, comment_id: comment.id, errors: e.message } - end - end - - def pull_request_comment_attributes(comment) - author = uid(comment) - note = '' - - unless author - author = project.creator_id - note = "*By #{comment.author_username} (#{comment.author_email})*\n\n" - end - - note += - # Provide some context for replying - if comment.parent_comment - "> #{comment.parent_comment.note.truncate(80)}\n\n#{comment.note}" - else - comment.note - end - - { - project: project, - note: note, - author_id: author, - created_at: comment.created_at, - updated_at: comment.updated_at - } - end - - def metrics - @metrics ||= Gitlab::Import::Metrics.new(:bitbucket_server_importer, @project) - end - - def author_line(rep_object) - return '' if uid(rep_object) - - @formatter.author_line(rep_object.author) - end - - def author_id(rep_object) - uid(rep_object) || project.creator_id - end - - def uid(rep_object) - # We want this to only match either username or email depending on the flag state. - # There should be no fall-through. - if Feature.enabled?(:bitbucket_server_user_mapping_by_username, type: :ops) - find_user_id(by: :username, value: rep_object.author_username) - else - find_user_id(by: :email, value: rep_object.author_email) - end - end - - def reviewers(reviewers) - return [] unless reviewers.present? - - reviewers.filter_map do |reviewer| - if Feature.enabled?(:bitbucket_server_user_mapping_by_username, type: :ops) - find_user_id(by: :username, value: reviewer.dig('user', 'slug')) - else - find_user_id(by: :email, value: reviewer.dig('user', 'emailAddress')) - end - end - end - end - end -end diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index 15178597a99..d0644856105 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -26,8 +26,10 @@ module Gitlab # sessions, replication lag could erroneously cause step 5 to # report no matching merge requests. To avoid this, we check # the write location to ensure the replica can make this query. + # Adding use_primary_on_empty_location: true for extra precaution in case there happens to be + # no LSN saved for the project then we will use the primary. track_session_metrics do - ::ApplicationRecord.sticking.select_valid_host(:project, @project.id) + ::ApplicationRecord.sticking.find_caught_up_replica(:project, @project.id, use_primary_on_empty_location: true) end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 42a8b561d34..d581fbfda42 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -299,7 +299,7 @@ module Gitlab end def handle_new_line - write_in_tag %{<br/>} + write_in_tag %(<br/>) close_open_tags if @sections.any? && @lineno_in_section == 0 @lineno_in_section += 1 @@ -324,7 +324,7 @@ module Gitlab return if @sections.include?(section) @sections << section - write_raw %{<div class="section-start" data-timestamp="#{timestamp}" data-section="#{data_section_names}" role="button"></div>} + write_raw %(<div class="section-start" data-timestamp="#{timestamp}" data-section="#{data_section_names}" role="button"></div>) @lineno_in_section = 0 end @@ -333,7 +333,7 @@ module Gitlab # close all sections up to section until @sections.empty? - write_raw %{<div class="section-end" data-section="#{data_section_names}"></div>} + write_raw %(<div class="section-end" data-section="#{data_section_names}"></div>) last_section = @sections.pop break if section == last_section @@ -423,9 +423,9 @@ module Gitlab close_open_tags @out << if css_classes.any? - %{<span class="#{css_classes.join(' ')}">} + %(<span class="#{css_classes.join(' ')}">) else - %{<span>} + %(<span>) end @n_open_tags += 1 @@ -433,7 +433,7 @@ module Gitlab def close_open_tags while @n_open_tags > 0 - @out << %{</span>} + @out << %(</span>) @n_open_tags -= 1 end end diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index d0ab4916c90..5748b8e34cf 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -18,7 +18,11 @@ module Gitlab def initialize(stream, path, **opts) @stream = stream - @path = path + + # Ensure to remove any ./ prefix from the path + # so that the pattern matching will work as expected + @path = path.gsub(%r{^\./}, '') + @opts = opts @full_version = read_version end @@ -59,7 +63,7 @@ module Gitlab entries = {} child_pattern = '[^/]*/?$' unless @opts[:recursive] - match_pattern = /^#{Regexp.escape(@path)}#{child_pattern}/ + match_pattern = /^#{Regexp.escape(path)}#{child_pattern}/ until gz.eof? begin diff --git a/lib/gitlab/ci/build/duration_parser.rb b/lib/gitlab/ci/build/duration_parser.rb index 9385dccd5f3..97049a4f876 100644 --- a/lib/gitlab/ci/build/duration_parser.rb +++ b/lib/gitlab/ci/build/duration_parser.rb @@ -41,7 +41,7 @@ module Gitlab def parse return if never? - ChronicDuration.parse(value) + ChronicDuration.parse(value, use_complete_matcher: true) end def validation_cache diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb index e0ef598da1b..17c784c4d54 100644 --- a/lib/gitlab/ci/components/instance_path.rb +++ b/lib/gitlab/ci/components/instance_path.rb @@ -7,17 +7,19 @@ module Gitlab include Gitlab::Utils::StrongMemoize LATEST_VERSION_KEYWORD = '~latest' + TEMPLATES_DIR = 'templates' def self.match?(address) address.include?('@') && address.start_with?(Settings.gitlab_ci['component_fqdn']) end - attr_reader :host + attr_reader :host, :project_file_path def initialize(address:, content_filename:) @full_path, @version = address.to_s.split('@', 2) @content_filename = content_filename @host = Settings.gitlab_ci['component_fqdn'] + @project_file_path = nil end def fetch_content!(current_user:) @@ -26,7 +28,7 @@ module Gitlab raise Gitlab::Access::AccessDeniedError unless Ability.allowed?(current_user, :download_code, project) - project.repository.blob_data_at(sha, project_file_path) + content(simple_template_path) || content(complex_template_path) || content(legacy_template_path) end def project @@ -34,13 +36,6 @@ module Gitlab end strong_memoize_attr :project - def project_file_path - return unless project - - component_dir = instance_path.delete_prefix(project.full_path) - File.join(component_dir, @content_filename).delete_prefix('/') - end - def sha return unless project return latest_version_sha if version == LATEST_VERSION_KEYWORD @@ -57,6 +52,11 @@ module Gitlab @full_path.delete_prefix(host) end + def component_path + instance_path.delete_prefix(project.full_path).delete_prefix('/') + end + strong_memoize_attr :component_path + # Given a path like "my-org/sub-group/the-project/path/to/component" # find the project "my-org/sub-group/the-project" by looking at all possible paths. def find_project_by_component_path(path) @@ -75,6 +75,36 @@ module Gitlab def latest_version_sha project.releases.latest&.sha end + + # A simple template consists of a single file + def simple_template_path + # Extract this line and move to fetch_content once we remove legacy fetching + return unless templates_dir_exists? && component_path.index('/').nil? + + @project_file_path = File.join(TEMPLATES_DIR, "#{component_path}.yml") + end + + # A complex template is directory-based and may consist of multiple files. + # Given a path like "my-org/sub-group/the-project/templates/component" + # returns the entry point path: "templates/component/template.yml". + def complex_template_path + # Extract this line and move to fetch_content once we remove legacy fetching + return unless templates_dir_exists? && component_path.index('/').nil? + + @project_file_path = File.join(TEMPLATES_DIR, component_path, @content_filename) + end + + def legacy_template_path + @project_file_path = File.join(component_path, @content_filename).delete_prefix('/') + end + + def templates_dir_exists? + project.repository.tree.trees.map(&:name).include?(TEMPLATES_DIR) + end + + def content(path) + project.repository.blob_data_at(sha, path) + end end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 0c293c3f0ef..73d329930a5 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -161,6 +161,7 @@ module Gitlab def build_context(project:, pipeline:, sha:, user:, parent_pipeline:, pipeline_config:) Config::External::Context.new( project: project, + pipeline: pipeline, sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index ee99354cb28..1119afab24a 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -51,7 +51,7 @@ module Gitlab entry :parallel, Entry::Product::Parallel, description: 'Parallel configuration for this job.', inherit: false, - metadata: { allowed_strategies: %i(matrix) } + metadata: { allowed_strategies: %i[matrix] } attributes :when, :allow_failure, :parallel diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb index e996b6b1312..476b928e471 100644 --- a/lib/gitlab/ci/config/entry/default.rb +++ b/lib/gitlab/ci/config/entry/default.rb @@ -14,7 +14,7 @@ module Gitlab include ::Gitlab::Config::Entry::Inheritable ALLOWED_KEYS = %i[before_script after_script hooks cache image services - interruptible timeout retry tags artifacts].freeze + interruptible timeout retry tags artifacts id_tokens].freeze validations do validates :config, allowed_keys: ALLOWED_KEYS @@ -65,6 +65,11 @@ module Gitlab description: 'Default artifacts.', inherit: false + entry :id_tokens, ::Gitlab::Config::Entry::ComposableHash, + description: 'Configured JWTs for this job', + inherit: false, + metadata: { composable_class: ::Gitlab::Ci::Config::Entry::IdToken } + private def overwrite_entry(deps, key, current_entry) diff --git a/lib/gitlab/ci/config/entry/include/rules.rb b/lib/gitlab/ci/config/entry/include/rules.rb index 71418e6752d..a3799b36ece 100644 --- a/lib/gitlab/ci/config/entry/include/rules.rb +++ b/lib/gitlab/ci/config/entry/include/rules.rb @@ -19,11 +19,6 @@ module Gitlab validates :config, type: Array end - # Remove this method when FF `ci_refactor_external_rules` is removed - def value - Feature.enabled?(:ci_refactor_external_rules) ? super : @config - end - def composable_class Entry::Include::Rules::Rule end diff --git a/lib/gitlab/ci/config/entry/include/rules/rule.rb b/lib/gitlab/ci/config/entry/include/rules/rule.rb index 1a68e95913c..df8509eecc0 100644 --- a/lib/gitlab/ci/config/entry/include/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/include/rules/rule.rb @@ -7,13 +7,17 @@ module Gitlab class Include class Rules::Rule < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[if exists when].freeze + ALLOWED_KEYS = %i[if exists when changes].freeze ALLOWED_WHEN = %w[never always].freeze attributes :if, :exists, :when + entry :changes, Entry::Rules::Rule::Changes, + description: 'File change condition rule.' + validations do validates :config, presence: true validates :config, type: { with: Hash } @@ -27,7 +31,9 @@ module Gitlab end def value - Feature.enabled?(:ci_refactor_external_rules) ? config.compact : super + config.merge( + changes: (changes_value if changes_defined?) + ).compact end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index d31d1b366c3..c40d665f320 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -13,7 +13,7 @@ module Gitlab ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze ALLOWED_KEYS = %i[tags script image services start_in artifacts cache dependencies before_script after_script hooks - environment coverage retry parallel interruptible timeout + coverage retry parallel interruptible timeout release id_tokens publish].freeze validations do @@ -102,10 +102,6 @@ module Gitlab metadata: { allowed_needs: %i[job cross_dependency] }, inherit: false - entry :environment, Entry::Environment, - description: 'Environment configuration for this job.', - inherit: false - entry :coverage, Entry::Coverage, description: 'Coverage configuration for this job.', inherit: false @@ -124,7 +120,7 @@ module Gitlab entry :id_tokens, ::Gitlab::Config::Entry::ComposableHash, description: 'Configured JWTs for this job', - inherit: false, + inherit: true, metadata: { composable_class: ::Gitlab::Ci::Config::Entry::IdToken } entry :publish, Entry::Publish, @@ -160,13 +156,11 @@ module Gitlab when: self.when, start_in: self.start_in, dependencies: dependencies, - environment: environment_defined? ? environment_value : nil, - environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, retry: retry_defined? ? retry_value : nil, parallel: has_parallel? ? parallel_value : nil, interruptible: interruptible_defined? ? interruptible_value : nil, - timeout: has_timeout? ? ChronicDuration.parse(timeout.to_s) : nil, + timeout: parsed_timeout, artifacts: artifacts_value, release: release_value, after_script: after_script_value, @@ -180,6 +174,12 @@ module Gitlab ).compact end + def parsed_timeout + return unless has_timeout? + + ChronicDuration.parse(timeout.to_s, use_complete_matcher: true) + end + def ignored? allow_failure_defined? ? static_allow_failure : manual_action? end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index e0f0903174c..88734ac1186 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -15,7 +15,7 @@ module Gitlab include ::Gitlab::Config::Entry::Inheritable PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables - inherit allow_failure when needs resource_group].freeze + inherit allow_failure when needs resource_group environment].freeze MAX_NESTING_LEVEL = 10 included do @@ -68,6 +68,10 @@ module Gitlab inherit: false, default: {} + entry :environment, Entry::Environment, + description: 'Environment configuration for this job.', + inherit: false + attributes :extends, :rules, :resource_group end @@ -125,6 +129,8 @@ module Gitlab root_variables_inheritance: root_variables_inheritance, only: only_value, except: except_value, + environment: environment_defined? ? environment_value : nil, + environment_name: environment_defined? ? environment_value[:name] : nil, resource_group: resource_group }.compact end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index c57391d355c..0a524fdba66 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -9,22 +9,21 @@ module Gitlab TimeoutError = Class.new(StandardError) - TEMP_MAX_INCLUDES = 100 # For logging; to be removed in https://gitlab.com/gitlab-org/gitlab/-/issues/396776 - include ::Gitlab::Utils::StrongMemoize attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config - attr_reader :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes + attr_reader :pipeline, :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes attr_accessor :total_file_size_in_bytes delegate :instrument, to: :logger def initialize( - project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, + project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, pipeline_config: nil, logger: nil ) @project = project + @pipeline = pipeline @sha = sha @user = user @parent_pipeline = parent_pipeline @@ -60,6 +59,7 @@ module Gitlab def mutate(attrs = {}) self.class.new(**attrs) do |ctx| + ctx.pipeline = pipeline ctx.expandset = expandset ctx.execution_deadline = execution_deadline ctx.logger = logger @@ -106,7 +106,7 @@ module Gitlab protected - attr_writer :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes + attr_writer :pipeline, :expandset, :execution_deadline, :logger, :max_includes, :max_total_yaml_size_bytes private diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb index 15cc0783b86..de6de1bb7a8 100644 --- a/lib/gitlab/ci/config/external/file/component.rb +++ b/lib/gitlab/ci/config/external/file/component.rb @@ -18,6 +18,8 @@ module Gitlab def content return unless component_result.success? + ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event('cicd_component_usage', values: context.user.id) + component_result.payload.fetch(:content) end strong_memoize_attr :content diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb index 580cae8a207..0e296aa0b5b 100644 --- a/lib/gitlab/ci/config/external/mapper/verifier.rb +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -40,7 +40,7 @@ module Gitlab file.validate_content! if file.valid? file.load_and_validate_expanded_hash! if file.valid? - next unless Feature.enabled?(:introduce_ci_max_total_yaml_size_bytes, context.project) && file.valid? + next unless file.valid? # We are checking the file.content.to_s because that is returning the actual content of the file, # whereas file.content would return the BatchLoader. diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb index 0e6209460e0..05266fbff0c 100644 --- a/lib/gitlab/ci/config/external/rules.rb +++ b/lib/gitlab/ci/config/external/rules.rb @@ -5,29 +5,19 @@ module Gitlab class Config module External class Rules - # Remove these two constants when FF `ci_refactor_external_rules` is removed - ALLOWED_KEYS = Entry::Include::Rules::Rule::ALLOWED_KEYS - ALLOWED_WHEN = Entry::Include::Rules::Rule::ALLOWED_WHEN - InvalidIncludeRulesError = Class.new(Mapper::Error) def initialize(rule_hashes) - if Feature.enabled?(:ci_refactor_external_rules) - return unless rule_hashes - - # We must compose the include rules entry here because included - # files are expanded before `@root.compose!` runs in Ci::Config. - rules_entry = Entry::Include::Rules.new(rule_hashes) - rules_entry.compose! + return unless rule_hashes - raise InvalidIncludeRulesError, "include:#{rules_entry.errors.first}" unless rules_entry.valid? + # We must compose the include rules entry here because included + # files are expanded before `@root.compose!` runs in Ci::Config. + rules_entry = Entry::Include::Rules.new(rule_hashes) + rules_entry.compose! - @rule_list = Build::Rules::Rule.fabricate_list(rules_entry.value) - else - validate(rule_hashes) + raise InvalidIncludeRulesError, "include:#{rules_entry.errors.first}" unless rules_entry.valid? - @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes) - end + @rule_list = Build::Rules::Rule.fabricate_list(rules_entry.value) end def evaluate(context) @@ -38,28 +28,14 @@ module Gitlab else Result.new('never') end + rescue Build::Rules::Rule::Clause::ParseError => e + raise InvalidIncludeRulesError, "include:#{e.message}" end private def match_rule(context) - @rule_list.find { |rule| rule.matches?(nil, context) } - end - - # Remove this method when FF `ci_refactor_external_rules` is removed - def validate(rule_hashes) - return unless rule_hashes.is_a?(Array) - - rule_hashes.each do |rule_hash| - next if (rule_hash.keys - ALLOWED_KEYS).empty? && valid_when?(rule_hash) - - raise InvalidIncludeRulesError, "invalid include rule: #{rule_hash}" - end - end - - # Remove this method when FF `ci_refactor_external_rules` is removed - def valid_when?(rule_hash) - rule_hash[:when].nil? || rule_hash[:when].in?(ALLOWED_WHEN) + @rule_list.find { |rule| rule.matches?(context.pipeline, context) } end Result = Struct.new(:when) do diff --git a/lib/gitlab/ci/config/interpolation/interpolator.rb b/lib/gitlab/ci/config/interpolation/interpolator.rb index 58965890184..95c419d7427 100644 --- a/lib/gitlab/ci/config/interpolation/interpolator.rb +++ b/lib/gitlab/ci/config/interpolation/interpolator.rb @@ -37,7 +37,12 @@ module Gitlab def interpolate! return @errors.push(config.error) unless config.valid? - return @errors.push('unknown input arguments') if inputs_without_header? + + if inputs_without_header? + return @errors.push( + _('Given inputs not defined in the `spec` section of the included configuration file')) + end + return @result ||= config.content unless config.has_header? return @errors.concat(header.errors) unless header.valid? diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index 8c730a9548f..29beba4774a 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -61,10 +61,6 @@ module Gitlab pipeline_source: pipeline.source&.to_sym, pipeline_source_bridge: pipeline.source_bridge ) - rescue StandardError => e - # We don't want endpoints relying on this code to fail if there's an error here. - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, pipeline_id: pipeline.id) - nil end strong_memoize_attr(:project_config) diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb index bc62fbe55ec..1e5200e8682 100644 --- a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb @@ -58,6 +58,15 @@ module Gitlab properties = data.dig('metadata', 'properties') source = CyclonedxProperties.parse_source(properties) report.set_source(source) if source + + tools = data.dig('metadata', 'tools') + authors = data.dig('metadata', 'authors') + + report.metadata = ::Gitlab::Ci::Reports::Sbom::Metadata.new.tap do |metadata| + metadata.tools = tools if tools + metadata.authors = authors if authors + metadata.properties = properties if properties + end end def parse_components diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 4bc2f6c7be7..cc3aa33e93b 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -128,10 +128,6 @@ module Gitlab .observe({ plan: project.actual_plan_name }, jobs_count) end - def observe_pipeline_includes_count(pipeline) - logger.observe(:pipeline_includes_count, pipeline.config_metadata&.[](:includes)&.count, once: true) - end - def increment_pipeline_failure_reason_counter(reason) metrics.pipeline_failure_reason_counter .increment(reason: (reason || :unknown_failure).to_s) diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index dd097187955..de147914850 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -30,7 +30,6 @@ module Gitlab @command.observe_creation_duration(current_monotonic_time - @start) @command.observe_pipeline_size(@pipeline) @command.observe_jobs_count_in_alive_pipelines - @command.observe_pipeline_includes_count(@pipeline) @pipeline end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index a18542288c9..db1b53e52e0 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -14,7 +14,6 @@ module Gitlab METRICS_SHARD_TAG_PREFIX = 'metrics_shard::' DEFAULT_METRICS_SHARD = 'default' - JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5 OPERATION_COUNTERS = [ :build_can_pick, @@ -57,7 +56,7 @@ module Gitlab def register_success(job) labels = { shared_runner: runner.instance_type?, - jobs_running_for_project: jobs_running_for_project(job), + jobs_running_for_project: job.project_jobs_running_on_instance_runners_count, shard: DEFAULT_METRICS_SHARD } if runner.instance_type? @@ -65,7 +64,7 @@ module Gitlab labels[:shard] = shard.gsub(METRICS_SHARD_TAG_PREFIX, '') if shard end - self.class.job_queue_duration_seconds.observe(labels, Time.current - job.queued_at) unless job.queued_at.nil? + self.class.job_queue_duration_seconds.observe(labels, job.time_in_queue_seconds) unless job.queued_at.nil? self.class.attempt_counter.increment end @@ -231,28 +230,6 @@ module Gitlab Gitlab::Metrics.histogram(name, comment, labels, buckets) end end - - private - - # rubocop: disable CodeReuse/ActiveRecord - def jobs_running_for_project(job) - return '+Inf' unless runner.instance_type? - - # excluding currently started job - running_jobs_count = running_jobs_relation(job) - .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1 - - if running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET - running_jobs_count - else - "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+" - end - end - - def running_jobs_relation(job) - ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) - end - # rubocop: enable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb index aba2d2e8b19..edcb17a61a7 100644 --- a/lib/gitlab/ci/reports/codequality_reports.rb +++ b/lib/gitlab/ci/reports/codequality_reports.rb @@ -6,7 +6,7 @@ module Gitlab class CodequalityReports attr_reader :degradations, :error_message - SEVERITY_PRIORITIES = %w(blocker critical major minor info unknown).map.with_index.to_h.freeze # { "blocker" => 0, "critical" => 1 ... } + SEVERITY_PRIORITIES = %w[blocker critical major minor info unknown].map.with_index.to_h.freeze # { "blocker" => 0, "critical" => 1 ... } CODECLIMATE_SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'codeclimate.json').to_s def initialize diff --git a/lib/gitlab/ci/reports/sbom/component.rb b/lib/gitlab/ci/reports/sbom/component.rb index 51fd6af7bc4..59816e75b2c 100644 --- a/lib/gitlab/ci/reports/sbom/component.rb +++ b/lib/gitlab/ci/reports/sbom/component.rb @@ -7,7 +7,7 @@ module Gitlab class Component include Gitlab::Utils::StrongMemoize - attr_reader :component_type, :version + attr_reader :component_type, :version, :path def initialize(type:, name:, purl:, version:) @component_type = type @@ -31,12 +31,24 @@ module Gitlab end strong_memoize_attr :purl + def purl_type + purl.type + end + + def type + component_type + end + def name return @name unless purl [purl.namespace, purl.name].compact.join('/') end + def key + [name, version, purl&.type] + end + private def supported_component_type? diff --git a/lib/gitlab/ci/reports/sbom/metadata.rb b/lib/gitlab/ci/reports/sbom/metadata.rb new file mode 100644 index 00000000000..8945259d331 --- /dev/null +++ b/lib/gitlab/ci/reports/sbom/metadata.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Sbom + class Metadata + attr_accessor :tools, :authors, :properties, :timestamp + + def initialize(tools: [], authors: [], properties: []) + @tools = tools + @authors = authors + @properties = properties + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/sbom/report.rb b/lib/gitlab/ci/reports/sbom/report.rb index 51fa8ce0d2e..9a71c67388d 100644 --- a/lib/gitlab/ci/reports/sbom/report.rb +++ b/lib/gitlab/ci/reports/sbom/report.rb @@ -5,10 +5,24 @@ module Gitlab module Reports module Sbom class Report - attr_reader :components, :source, :errors + # This represents the attributes defined in cycloneDX Schema + # https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/validators/json_schemas/cyclonedx_report.json#L7 + BOM_FORMAT = 'CycloneDX' + SPEC_VERSION = '1.4' + VERSION = 1 + + attr_reader :source, :errors + attr_accessor :sbom_attributes, :metadata, :components def initialize + @sbom_attributes = { + bom_format: BOM_FORMAT, + spec_version: SPEC_VERSION, + serial_number: "urn:uuid:#{SecureRandom.uuid}", + version: VERSION + } @components = [] + @metadata = ::Gitlab::Ci::Reports::Sbom::Metadata.new @errors = [] end diff --git a/lib/gitlab/ci/reports/test_reports_comparer.rb b/lib/gitlab/ci/reports/test_reports_comparer.rb index 497831ae5a7..0ea2d793eea 100644 --- a/lib/gitlab/ci/reports/test_reports_comparer.rb +++ b/lib/gitlab/ci/reports/test_reports_comparer.rb @@ -29,7 +29,7 @@ module Gitlab end end - %w(total_count resolved_count failed_count error_count).each do |method| + %w[total_count resolved_count failed_count error_count].each do |method| define_method(method) do # rubocop: disable CodeReuse/ActiveRecord suite_comparers.sum { |suite| suite.public_send(method) } # rubocop:disable GitlabSecurity/PublicSend diff --git a/lib/gitlab/ci/reports/test_suite.rb b/lib/gitlab/ci/reports/test_suite.rb index dcc593b4403..c681727a43d 100644 --- a/lib/gitlab/ci/reports/test_suite.rb +++ b/lib/gitlab/ci/reports/test_suite.rb @@ -77,7 +77,7 @@ module Gitlab def +(other) self.class.new.tap do |test_suite| - test_suite.name = self.name + test_suite.name = other.name test_suite.test_cases = self.test_cases.deep_merge(other.test_cases) test_suite.total_time = self.total_time + other.total_time end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 6e2faf33a2f..fa1d8bec7e6 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -65,6 +65,10 @@ variables: DOCKER_TLS_CERTDIR: "" # https://gitlab.com/gitlab-org/gitlab-runner/issues/4501 + # License-Scanning job is removed from GitLab 16.3 + # This is the fix for https://gitlab.com/gitlab-org/gitlab/-/issues/422791 + LICENSE_MANAGEMENT_DISABLED: "true" + stages: - build - test diff --git a/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml b/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml new file mode 100644 index 00000000000..48c9422b469 --- /dev/null +++ b/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml @@ -0,0 +1,22 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml + +# This template extends Docker.gitlab-ci.yml to sign the image with Cosign after building. +# This allows you to verify that an image was built by a trusted pipeline before running it. +# See https://docs.gitlab.com/ee/ci/yaml/signing_examples.html for more details. + +include: + template: Docker.gitlab-ci.yml + +docker-build: + variables: + COSIGN_YES: "true" # Used by Cosign to skip confirmation prompts for non-destructive operations + id_tokens: + SIGSTORE_ID_TOKEN: # Used by Cosign to get certificate from Fulcio + aud: sigstore + after_script: + - apk add --update cosign + - IMAGE_DIGEST="$(docker inspect --format='{{index .RepoDigests 0}}' "$DOCKER_IMAGE_NAME")" + - cosign sign "$IMAGE_DIGEST" diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml index 8f5f0e2c451..1aa346aec67 100644 --- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -15,21 +15,20 @@ docker-build: stage: build services: - docker:dind + variables: + DOCKER_IMAGE_NAME: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - # Default branch leaves tag empty (= latest tag) - # All other branches are tagged with the escaped branch name (commit ref slug) + # All branches are tagged with $DOCKER_IMAGE_NAME (defaults to commit ref slug) + # Default branch is also tagged with `latest` script: + - docker build --pull -t "$DOCKER_IMAGE_NAME" . + - docker push "$DOCKER_IMAGE_NAME" - | if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then - tag="" - echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" - else - tag=":$CI_COMMIT_REF_SLUG" - echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + docker tag "$DOCKER_IMAGE_NAME" "$CI_REGISTRY_IMAGE:latest" + docker push "$CI_REGISTRY_IMAGE:latest" fi - - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . - - docker push "$CI_REGISTRY_IMAGE${tag}" # Run this job in a branch where a Dockerfile exists rules: - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index c1aedbe1111..07bc3fbe795 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.38.1' + AUTO_BUILD_IMAGE_VERSION: 'v1.41.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index c1aedbe1111..07bc3fbe795 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.38.1' + AUTO_BUILD_IMAGE_VERSION: 'v1.41.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml index 192d06bfa14..5cee19a746c 100644 --- a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml @@ -40,6 +40,7 @@ container_scanning: reports: container_scanning: gl-container-scanning-report.json dependency_scanning: gl-dependency-scanning-report.json + cyclonedx: "**/gl-sbom-*.cdx.json" paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json, "**/gl-sbom-*.cdx.json"] dependencies: [] script: diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml index 9a4c75e7402..ade4be99f18 100644 --- a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml @@ -40,6 +40,7 @@ container_scanning: reports: container_scanning: gl-container-scanning-report.json dependency_scanning: gl-dependency-scanning-report.json + cyclonedx: "**/gl-sbom-*.cdx.json" paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json, "**/gl-sbom-*.cdx.json"] dependencies: [] script: diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 7b2fb49b65e..e9ba938142d 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.53.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.56.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 1e482ccca82..eaaf171e4b5 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.53.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.56.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 6eac691b293..d2e448fb6a1 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.53.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.56.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml b/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml index 30767e66649..1468cf9c7c6 100644 --- a/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml @@ -3,17 +3,17 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/MATLAB.gitlab-ci.yml -# Use this template to run MATLAB and Simulink as part of your CI/CD pipeline. The template includes three jobs: +# Use this template to build and test your MATLAB project as part of your CI/CD pipeline. The template includes four jobs: # - `command`: Run MATLAB scripts, functions, and statements. # - `test`: Run tests authored using the MATLAB unit testing framework or Simulink Test. # - `test_artifacts`: Run MATLAB and Simulink tests, and generate test and coverage artifacts. +# - `build`: Run a build using the MATLAB build tool. # # The jobs in the template use the `matlab -batch` syntax to start MATLAB. The `-batch` option is supported # in MATLAB R2019a and later. # # You can copy and paste one or more jobs in this template into your `.gitlab-ci.yml` file. # You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. -# # Your runner must use the Docker executor to run MATLAB within a container. The [MATLAB Container on Docker Hub][1] # lets you run your build using MATLAB R2020b or a later release. If your build requires additional toolboxes, use a @@ -24,7 +24,7 @@ # [2] https://www.mathworks.com/help/cloudcenter/ug/create-a-custom-matlab-container.html # The jobs in this template incorporate the contents of a hidden `.matlab_defaults` job. You need to -# configure this job before running the `command`, `test`, and `test_artifacts` jobs. To configure the job: +# configure this job before running the `command`, `test`, `test_artifacts`, and `build` jobs. To configure the job: # - Specify the name of the MATLAB container image you want to use. # - Set the `MLM_LICENSE_FILE` environment variable using the port number and DNS address for your network license manager. # @@ -40,17 +40,17 @@ # command: extends: .matlab_defaults - script: matlab -batch mycommand + script: matlab -batch "mycommand" # If you specify more than one script, function, or statement, use a comma or semicolon to separate them. # For example, to run `myscript.m` in a folder named `myfolder` located in the root of the repository, -# you can specify `mycommand` like this: +# you can specify `"mycommand"` like this: # # "addpath('myfolder'), myscript" # # MATLAB exits with exit code 0 if the specified script, function, or statement executes successfully without # error. Otherwise, MATLAB terminates with a nonzero exit code, which causes the job to fail. To have the -# job fail in certain conditions, use the [`assert`][3] or [`error`][4] functions. +# job fail in certain conditions, use the [`assert`][3] or [`error`][4] function. # # [3] https://www.mathworks.com/help/matlab/ref/assert.html # [4] https://www.mathworks.com/help/matlab/ref/error.html @@ -62,7 +62,7 @@ test: extends: .matlab_defaults script: matlab -batch "results = runtests('IncludeSubfolders',true), assertSuccess(results);" -# By default, the job includes any files in your [MATLAB Project][7] that have a `Test` label. If your repository +# By default, the job includes any files in your [MATLAB project][7] that have a `Test` label. If your repository # does not have a MATLAB project, then the job includes all tests in the root of your repository or in any of # its subfolders. # @@ -71,9 +71,9 @@ test: # [7] https://www.mathworks.com/help/matlab/projects.html # The `test_artifacts` job runs your tests and additionally generates test and coverage artifacts. -# It uses the plugin classes in the [`matlab.unittest.plugins`][8] package to generate a JUnit test results -# report and a Cobertura code coverage report. Like the `test` job, this job runs all the tests in your -# project and fails the build if any of the tests fail. +# It uses the plugin classes in the [`matlab.unittest.plugins`][8] package to produce test results +# in JUnit-style XML format and code coverage results in Cobertura XML format. Like the `test` job, +# this job runs all the tests in your project and fails the build if any of the tests fail. # test_artifacts: extends: .matlab_defaults @@ -110,3 +110,22 @@ test_artifacts: # # [8] https://www.mathworks.com/help/matlab/ref/matlab.unittest.plugins-package.html # [9] https://www.mathworks.com/help/matlab/matlab_prog/generate-artifacts-using-matlab-unit-test-plugins.html + +# Starting in R2022b, the `build` job runs a build using the MATLAB build tool. You can use this job to run the +# tasks specified in a file named `buildfile.m` in the root of your repository. +# +build: + extends: .matlab_defaults + script: matlab -batch "buildtool" + +# The job executes the [`buildtool`][10] command to run a build using the default tasks in `buildfile.m` +# as well as all the tasks on which they depend. To run specific tasks instead, specify them as a space-separated +# list in the job. For example, to run the tasks named `task1` and `task2` and their dependencies, substitute +# `"buildtool"` with `"buildtool task1 task2"`. +# +# MATLAB exits with exit code 0 if the build runs successfully. Otherwise, MATLAB terminates with a nonzero +# exit code, which causes the job to fail. For more information about the MATLAB build tool, +# see [Create and Run Tasks Using Build Tool][11]. +# +# [10] https://www.mathworks.com/help/matlab/ref/buildtool.html +# [11] https://www.mathworks.com/help/matlab/matlab_prog/create-and-run-tasks-using-build-tool.html diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 2dc7bbc391e..f4ba9100812 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -30,9 +30,9 @@ module Gitlab @job = job end - def html(last_lines: nil) + def html(last_lines: nil, max_size: nil) read do |stream| - stream.html(last_lines: last_lines) + stream.html(last_lines: last_lines, max_size: max_size) end end @@ -290,7 +290,7 @@ module Gitlab if consistent_archived_trace?(build) ::Ci::Build .sticking - .unstick_or_continue_sticking(LOAD_BALANCING_STICKING_NAMESPACE, build.id) + .find_caught_up_replica(LOAD_BALANCING_STICKING_NAMESPACE, build.id) end yield diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb index dd435ba05b7..ef494a79d9a 100644 --- a/lib/gitlab/ci/trace/stream.rb +++ b/lib/gitlab/ci/trace/stream.rb @@ -53,18 +53,20 @@ module Gitlab append(data, 0) end - def raw(last_lines: nil) + def raw(last_lines: nil, max_size: nil) return unless valid? - if last_lines.to_i > 0 + if max_size.to_i > 0 + read_last_lines_with_max_size(last_lines, max_size) + elsif last_lines.to_i > 0 read_last_lines(last_lines) else stream.read end.force_encoding(Encoding.default_external) end - def html(last_lines: nil) - text = raw(last_lines: last_lines) + def html(last_lines: nil, max_size: nil) + text = raw(last_lines: last_lines, max_size: max_size) buffer = StringIO.new(text) ::Gitlab::Ci::Ansi2html.convert(buffer).html end @@ -117,6 +119,37 @@ module Gitlab to_enum(:reverse_line).first(limit).reverse.join end + def read_last_lines_with_max_size(limit, max_size) + linesleft = limit + result = '' + + reverse_line_with_max_size(max_size) do |line| + result = line + result + unless linesleft.nil? + linesleft -= 1 + break if linesleft <= 0 + end + end + + result + end + + def reverse_line_with_max_size(max_size) + stream.seek(0, IO::SEEK_END) + debris = '' + sizeleft = max_size + + until sizeleft <= 0 || (buf = read_backward([BUFFER_SIZE, sizeleft].min)).empty? + sizeleft -= buf.bytesize + debris, *lines = (buf + debris).each_line.to_a + lines.reverse_each do |line| + yield(line.force_encoding(Encoding.default_external)) + end + end + + yield(debris.force_encoding(Encoding.default_external)) unless debris.empty? + end + def reverse_line stream.seek(0, IO::SEEK_END) debris = '' diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index cae3a966bc6..c279af6acfc 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -17,7 +17,7 @@ module Gitlab def scoped_variables(job, environment:, dependencies:) Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.concat(predefined_variables(job)) + variables.concat(predefined_variables(job, environment)) variables.concat(project.predefined_variables) variables.concat(pipeline_variables_builder.predefined_variables) variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner @@ -126,7 +126,7 @@ module Gitlab delegate :project, to: :pipeline - def predefined_variables(job) + def predefined_variables(job, environment) Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI_JOB_NAME', value: job.name) variables.append(key: 'CI_JOB_NAME_SLUG', value: job_name_slug(job)) @@ -137,8 +137,12 @@ module Gitlab variables.append(key: 'CI_NODE_INDEX', value: job.options[:instance].to_s) if job.options&.include?(:instance) variables.append(key: 'CI_NODE_TOTAL', value: ci_node_total_value(job).to_s) - # Set environment name here so we can access it when evaluating the job's rules - variables.append(key: 'CI_ENVIRONMENT_NAME', value: job.environment) if job.environment + if environment.present? + variables.append(key: 'CI_ENVIRONMENT_NAME', value: environment) + variables.append(key: 'CI_ENVIRONMENT_ACTION', value: job.environment_action) + variables.append(key: 'CI_ENVIRONMENT_TIER', value: job.environment_tier) + variables.append(key: 'CI_ENVIRONMENT_URL', value: job.environment_url) if job.environment_url + end end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index c69d9218a66..3a0173d1548 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -129,6 +129,12 @@ module Gitlab error!("#{name} job: undefined #{dependency_type}: #{dependency}") end + # A parallel job's name is expanded in Config::Normalizer so we must revalidate the name length here + if dependency_type == 'need' && dependency.length > ::Ci::BuildNeed::MAX_JOB_NAME_LENGTH + error!("#{name} job: need `#{dependency}` name is too long " \ + "(maximum is #{::Ci::BuildNeed::MAX_JOB_NAME_LENGTH} characters)") + end + job_stage_index = stage_index(name) dependency_stage_index = stage_index(dependency) diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb index 90123b9d000..486d53eb870 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb @@ -9,7 +9,7 @@ module Gitlab LOST_AND_FOUND = File.join(ABSOLUTE_ARTIFACT_DIR, '-', 'lost+found').freeze BATCH_SIZE = 500 DEFAULT_NICENESS = 'best-effort' - VALID_NICENESS_LEVELS = %w{none realtime best-effort idle}.freeze + VALID_NICENESS_LEVELS = %w[none realtime best-effort idle].freeze attr_accessor :batch, :total_found, :total_cleaned attr_reader :dry_run, :niceness, :logger diff --git a/lib/gitlab/cluster/rack_timeout_observer.rb b/lib/gitlab/cluster/rack_timeout_observer.rb index 15dd6a59e19..8e9fc16c588 100644 --- a/lib/gitlab/cluster/rack_timeout_observer.rb +++ b/lib/gitlab/cluster/rack_timeout_observer.rb @@ -4,7 +4,7 @@ module Gitlab module Cluster class RackTimeoutObserver include ActionView::Helpers::SanitizeHelper - TRANSITION_STATES = %i(ready active).freeze + TRANSITION_STATES = %i[ready active].freeze def initialize @counter = Gitlab::Metrics.counter(:rack_requests_total, 'Number of requests in a given rack state') diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb index fdff8fb32d3..bfa3112b795 100644 --- a/lib/gitlab/composer/version_index.rb +++ b/lib/gitlab/composer/version_index.rb @@ -48,7 +48,8 @@ module Gitlab end def package_source(package) - git_url = package.project.http_url_to_repo + use_http_url = package.project.public? || Feature.disabled?(:composer_use_ssh_source_urls, package.project) + git_url = use_http_url ? package.project.http_url_to_repo : package.project.ssh_url_to_repo { 'type' => 'git', diff --git a/lib/gitlab/config/entry/legacy_validation_helpers.rb b/lib/gitlab/config/entry/legacy_validation_helpers.rb index 415f6f77214..ec67d65c526 100644 --- a/lib/gitlab/config/entry/legacy_validation_helpers.rb +++ b/lib/gitlab/config/entry/legacy_validation_helpers.rb @@ -12,7 +12,7 @@ module Gitlab if parser && parser.respond_to?(:validate_duration) parser.validate_duration(value) else - ChronicDuration.parse(value) + ChronicDuration.parse(value, use_complete_matcher: true) end rescue ChronicDuration::DurationParseError false @@ -24,8 +24,12 @@ module Gitlab if parser && parser.respond_to?(:validate_duration_limit) parser.validate_duration_limit(value, limit) else - ChronicDuration.parse(value).second.from_now < - ChronicDuration.parse(limit).second.from_now + ChronicDuration.parse( + value, use_complete_matcher: true + ).second.from_now < + ChronicDuration.parse( + limit, use_complete_matcher: true + ).second.from_now end rescue ChronicDuration::DurationParseError false diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 59a4e425b85..87a6d4ada70 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -29,6 +29,7 @@ module Gitlab allow_customersdot(directives) allow_review_apps(directives) csp_level_3_backport(directives) + add_browsersdk_tracking(directives) directives end @@ -83,6 +84,16 @@ module Gitlab append_to_directive(directives, 'connect_src', url) end + def add_browsersdk_tracking(directives) + return if directives.blank? + return unless Gitlab.com? && Feature.enabled?(:browsersdk_tracking) && ENV['GITLAB_ANALYTICS_URL'].present? + + default_connect_src = directives['connect-src'] || directives['default-src'] + connect_src_values = Array.wrap(default_connect_src) | [ENV['GITLAB_ANALYTICS_URL']] + + append_to_directive(directives, 'connect_src', connect_src_values.join(' ')) + end + def allow_lfs(directives) return unless Gitlab.config.lfs.enabled && LfsObjectUploader.object_store_enabled? && LfsObjectUploader.direct_download_enabled? diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index b26f9a61ee1..915d606feb9 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -9,7 +9,7 @@ module Gitlab def build(deployment, status, status_changed_at) # Deployments will not have a deployable when created using the API. deployable_url = - if deployment.deployable + if deployment.deployable.instance_of?(::Ci::Build) Gitlab::UrlBuilder.build(deployment.deployable) end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 89065c11c4f..6222155d812 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -185,37 +185,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def self.check_postgres_version_and_print_warning - return if Gitlab::Runtime.rails_runner? - - database_base_models.each do |name, model| - database = Gitlab::Database::Reflection.new(model) - - next if database.postgresql_minimum_supported_version? - - Kernel.warn ERB.new(Rainbow.new.wrap(<<~EOS).red).result - - ██ ██ █████ ██████ ███ ██ ██ ███ ██ ██████ - ██ ██ ██ ██ ██ ██ ████ ██ ██ ████ ██ ██ - ██ █ ██ ███████ ██████ ██ ██ ██ ██ ██ ██ ██ ██ ███ - ██ ███ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ - ███ ███ ██ ██ ██ ██ ██ ████ ██ ██ ████ ██████ - - ****************************************************************************** - You are using PostgreSQL #{database.version} for the #{name} database, but this version of GitLab requires PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %>. - <% if Rails.env.development? || Rails.env.test? %> - If using gitlab-development-kit, please find the relevant steps here: - https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#upgrade-postgresql - <% end %> - Please upgrade your environment to a supported PostgreSQL version. See - https://docs.gitlab.com/ee/install/requirements.html#database for details. - ****************************************************************************** - EOS - rescue ActiveRecord::ActiveRecordError, PG::Error - # ignore - happens when Rake tasks yet have to create a database, e.g. for testing - end - end - def self.random "RANDOM()" end diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb index 417e9f211b9..a6a7e7455e9 100644 --- a/lib/gitlab/database/as_with_materialized.rb +++ b/lib/gitlab/database/as_with_materialized.rb @@ -4,31 +4,15 @@ module Gitlab module Database # This class is a special Arel node which allows optionally define the `MATERIALIZED` keyword for CTE and Recursive CTE queries. class AsWithMaterialized < Arel::Nodes::As - extend Gitlab::Utils::StrongMemoize - MATERIALIZED = 'MATERIALIZED ' def initialize(left, right, materialized: true) - if materialized && self.class.materialized_supported? + if materialized right.prepend(MATERIALIZED) end super(left, right) end - - # Note: to be deleted after the minimum PG version is set to 12.0 - def self.materialized_supported? - strong_memoize(:materialized_supported) do - ApplicationRecord.database.version.match?(/^1[2-9]\./) # version 12.x and above - end - end - - # Note: to be deleted after the minimum PG version is set to 12.0 - # Update the documentation together when deleting the method - # https://docs.gitlab.com/ee/development/merge_request_concepts/performance.html#use-ctes-wisely - def self.materialized_if_supported - materialized_supported? ? 'MATERIALIZED' : '' - end end end end diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb index db05635c73d..28a61dad3b5 100644 --- a/lib/gitlab/database/async_indexes/migration_helpers.rb +++ b/lib/gitlab/database/async_indexes/migration_helpers.rb @@ -39,6 +39,7 @@ module Gitlab Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! return unless async_index_creation_available? + raise "Table #{table_name} does not exist" unless table_exists?(table_name) index_name = options[:name] || index_name(table_name, column_name) diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb index 36dbb157b0d..51f39419ddb 100644 --- a/lib/gitlab/database/bulk_update.rb +++ b/lib/gitlab/database/bulk_update.rb @@ -114,7 +114,7 @@ module Gitlab def sql <<~SQL - WITH cte(#{list_of(cte_columns)}) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (VALUES #{list_of(values)}) + WITH cte(#{list_of(cte_columns)}) AS MATERIALIZED (VALUES #{list_of(values)}) UPDATE #{table_name} SET #{list_of(updates)} FROM cte WHERE cte_id = id SQL end diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index 2c480eb2cdc..67cde3e687d 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -17,22 +17,22 @@ module Gitlab # These methods perform writes after which we need to stick to the # primary. - STICKY_WRITES = %i( + STICKY_WRITES = %i[ delete delete_all insert update update_all exec_insert_all - ).freeze + ].freeze - NON_STICKY_READS = %i( + NON_STICKY_READS = %i[ sanitize_limit select select_one select_rows quote_column_name - ).freeze + ].freeze # hosts - The hosts to use for load balancing. def initialize(load_balancer) diff --git a/lib/gitlab/database/load_balancing/host.rb b/lib/gitlab/database/load_balancing/host.rb index f8ed5fcd4cc..880324f3ae4 100644 --- a/lib/gitlab/database/load_balancing/host.rb +++ b/lib/gitlab/database/load_balancing/host.rb @@ -80,11 +80,26 @@ module Gitlab start_time = ::Gitlab::Metrics::System.monotonic_time while (::Gitlab::Metrics::System.monotonic_time - start_time) <= timeout - break if pool.connections.none?(&:in_use?) + return if try_disconnect sleep(2) end + force_disconnect! + end + + # Attempt to disconnect the pool if all connections are no longer in use. + # Returns true if the pool was disconnected, false if not. + def try_disconnect + if pool.connections.none?(&:in_use?) + pool.disconnect! + return true + end + + false + end + + def force_disconnect! pool.disconnect! end @@ -104,16 +119,19 @@ module Gitlab def online? return @online unless check_replica_status? + was_online = @online refresh_status - if @online + # Log that the host came back online if it was previously offline + if @online && !was_online ::Gitlab::Database::LoadBalancing::Logger.info( event: :host_online, message: 'Host is online after replica status check', db_host: @host, db_port: @port ) - else + # Always log if the host goes offline + elsif !@online ::Gitlab::Database::LoadBalancing::Logger.warn( event: :host_offline, message: 'Host is offline after replica status check', diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index f6144b7b772..9495648d069 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -8,6 +8,10 @@ module Gitlab # Each host in the load balancer uses the same credentials as the primary # database. class LoadBalancer + ANY_CAUGHT_UP = :any + ALL_CAUGHT_UP = :all + NONE_CAUGHT_UP = :none + CACHE_KEY = :gitlab_load_balancer_host REPLICA_SUFFIX = '_replica' @@ -178,16 +182,32 @@ module Gitlab raise 'Failed to determine the write location of the primary database' end - # Returns true if there was at least one host that has caught up with the given transaction. + # Finds any up to date replica for the given LSN location and stores an up to date replica in the + # SafeRequestStore to be used later for read-only queries. It returns a symbol to indicate if :any, :all or + # :none were found to be caught up. def select_up_to_date_host(location) all_hosts = @host_list.hosts.shuffle - host = all_hosts.find { |host| host.caught_up?(location) } + first_caught_up_host = nil + + # We must loop through all of them so that we know if all are caught up. Some callers only care about finding + # one caught up host and storing it in request_cache. But Sticking needs to know if ALL_CAUGHT_UP so that it + # can clear the LSN position from Redis and not ask again in future. + results = all_hosts.map do |host| + caught_up = host.caught_up?(location) + first_caught_up_host ||= host if caught_up + caught_up + end + + ActiveSupport::Notifications.instrument( + 'caught_up_replica_pick.load_balancing', + { result: first_caught_up_host.present? } + ) - return false unless host + return NONE_CAUGHT_UP unless first_caught_up_host - request_cache[CACHE_KEY] = host + request_cache[CACHE_KEY] = first_caught_up_host - true + results.all? ? ALL_CAUGHT_UP : ANY_CAUGHT_UP end # Yields a block, retrying it upon error using an exponential backoff. diff --git a/lib/gitlab/database/load_balancing/rack_middleware.rb b/lib/gitlab/database/load_balancing/rack_middleware.rb index 99b1c31b04b..554205c8dcc 100644 --- a/lib/gitlab/database/load_balancing/rack_middleware.rb +++ b/lib/gitlab/database/load_balancing/rack_middleware.rb @@ -18,7 +18,7 @@ module Gitlab # doesn't linger around. clear - unstick_or_continue_sticking(env) + find_caught_up_replica(env) result = @app.call(env) @@ -35,20 +35,22 @@ module Gitlab # # Typically this code will only be reachable for Rails requests as # Grape data is not yet available at this point. - def unstick_or_continue_sticking(env) + def find_caught_up_replica(env) namespaces_and_ids = sticking_namespaces(env) namespaces_and_ids.each do |(sticking, namespace, id)| - sticking.unstick_or_continue_sticking(namespace, id) + sticking.find_caught_up_replica(namespace, id) end end # Determine if we need to stick after handling a request. def stick_if_necessary(env) + return unless ::Gitlab::Database::LoadBalancing::Session.current.performed_write? + namespaces_and_ids = sticking_namespaces(env) namespaces_and_ids.each do |sticking, namespace, id| - sticking.stick_if_necessary(namespace, id) + sticking.stick(namespace, id) end end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index a0b0ad19f73..1f9ab1cfe98 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -118,7 +118,7 @@ module Gitlab # The return value is the amount of time (in seconds) to wait before # checking the DNS record for any changes. def refresh_if_necessary - interval, from_dns = addresses_from_dns + wait_time, from_dns = addresses_from_dns current = addresses_from_load_balancer @@ -132,7 +132,7 @@ module Gitlab replace_hosts(from_dns) end - interval + wait_time end # Replaces all the hosts in the load balancer with the new ones, @@ -151,8 +151,12 @@ module Gitlab # started just before we added the new hosts it will use an old # host/connection. While this connection will be checked in and out, # it won't be explicitly disconnected. - old_hosts.each do |host| - host.disconnect!(timeout: disconnect_timeout) + if Gitlab::Utils.to_boolean(ENV['LOAD_BALANCER_PARALLEL_DISCONNECT'], default: false) + disconnect_old_hosts(old_hosts) + else + old_hosts.each do |host| + host.disconnect!(timeout: disconnect_timeout) + end end end @@ -250,6 +254,41 @@ module Gitlab @sampler ||= ::Gitlab::Database::LoadBalancing::ServiceDiscovery::Sampler .new(max_replica_pools: @max_replica_pools) end + + def disconnect_old_hosts(hosts) + return unless hosts.present? + + gentle_disconnect_start = ::Gitlab::Metrics::System.monotonic_time + gentle_disconnect_deadline = gentle_disconnect_start + disconnect_timeout + + hosts_to_disconnect = hosts + + gentle_disconnect_duration = Benchmark.realtime do + while ::Gitlab::Metrics::System.monotonic_time < gentle_disconnect_deadline + hosts_to_disconnect = hosts_to_disconnect.reject(&:try_disconnect) + + break if hosts_to_disconnect.empty? + + sleep(2) + end + end + + force_disconnect_duration = Benchmark.realtime do + # This may wait up to 2 * pool.checkout_timeout per host (default 10 seconds per host) + hosts_to_disconnect.each(&:force_disconnect!) + end + + formatted_hosts = hosts_to_disconnect.map { |h| "#{h.host}:#{h.port}" } + total_disconnect_duration = gentle_disconnect_duration + force_disconnect_duration + + ::Gitlab::Database::LoadBalancing::Logger.info( + event: :host_list_disconnection, + message: "Disconnected #{formatted_hosts} old load balancing hosts after #{total_disconnect_duration}s", + gentle_disconnect_duration_s: gentle_disconnect_duration, + force_disconnect_duration_s: force_disconnect_duration, + total_disconnect_duration_s: total_disconnect_duration + ) + end end end end diff --git a/lib/gitlab/database/load_balancing/service_discovery/sampler.rb b/lib/gitlab/database/load_balancing/service_discovery/sampler.rb index 71870214156..4fc11cc1035 100644 --- a/lib/gitlab/database/load_balancing/service_discovery/sampler.rb +++ b/lib/gitlab/database/load_balancing/service_discovery/sampler.rb @@ -16,7 +16,7 @@ module Gitlab def sample(addresses) return addresses if @max_replica_pools.nil? || addresses.count <= @max_replica_pools - ::Gitlab::Database::LoadBalancing::Logger.info( + ::Gitlab::Database::LoadBalancing::Logger.debug( event: :host_list_limit_exceeded, message: "Host list length exceeds max_replica_pools so random hosts will be chosen.", max_replica_pools: @max_replica_pools, diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb index f5cb83e398a..df07f510c8d 100644 --- a/lib/gitlab/database/load_balancing/sticking.rb +++ b/lib/gitlab/database/load_balancing/sticking.rb @@ -14,81 +14,38 @@ module Gitlab @load_balancer = load_balancer end - # Unsticks or continues sticking the current request. - # - # This method also updates the Rack environment so #call can later - # determine if we still need to stick or not. - # - # env - The Rack environment. - # namespace - The namespace to use for sticking. - # id - The identifier to use for sticking. - # model - The ActiveRecord model to scope sticking to. - def stick_or_unstick_request(env, namespace, id) - unstick_or_continue_sticking(namespace, id) - - env[::Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT] ||= Set.new - env[::Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT] << [self, namespace, id] - end - - # Sticks to the primary if a write was performed. - def stick_if_necessary(namespace, id) - stick(namespace, id) if ::Gitlab::Database::LoadBalancing::Session.current.performed_write? - end - - def all_caught_up?(namespace, id) - location = last_write_location_for(namespace, id) - - return true unless location - - @load_balancer.select_up_to_date_host(location).tap do |found| - ActiveSupport::Notifications.instrument( - 'caught_up_replica_pick.load_balancing', - { result: found } - ) - - unstick(namespace, id) if found - end - end - - # Selects hosts that have caught up with the primary. This ensures - # atomic selection of the host to prevent the host list changing - # in another thread. - # - # Returns true if one host was selected. - def select_caught_up_replicas(namespace, id) + # Returns true if any caught up replica is found. This does not mean all replicas are caught up but the found + # caught up replica will be stored in the SafeRequestStore available as LoadBalancer#host for future queries. + # With use_primary_on_empty_location: true we will assume you need the primary if we can't find a matching + # location for the namespace, id pair. You should only use use_primary_on_empty_location in rare cases because + # we unstick once we find all replicas are caught up one time so it can be wasteful on the primary. + def find_caught_up_replica(namespace, id, use_primary_on_failure: true, use_primary_on_empty_location: false) location = last_write_location_for(namespace, id) - # Unlike all_caught_up?, we return false if no write location exists. - # We want to be sure we talk to a replica that has caught up for a specific - # write location. If no such location exists, err on the side of caution. - return false unless location - - @load_balancer.select_up_to_date_host(location).tap do |selected| - unstick(namespace, id) if selected - end - end + result = if location + up_to_date_result = @load_balancer.select_up_to_date_host(location) - # Sticks to the primary if necessary, otherwise unsticks an object (if - # it was previously stuck to the primary). - def unstick_or_continue_sticking(namespace, id) - return if all_caught_up?(namespace, id) + unstick(namespace, id) if up_to_date_result == LoadBalancer::ALL_CAUGHT_UP - ::Gitlab::Database::LoadBalancing::Session.current.use_primary! - end + up_to_date_result != LoadBalancer::NONE_CAUGHT_UP + else + # Some callers want to err on the side of caution and be really sure that a caught up replica was + # found. If we did not have any location to check then we must force `use_primary!` if they they + # use_primary_on_empty_location + !use_primary_on_empty_location + end - # Select a replica that has caught up with the primary. If one has not been - # found, stick to the primary. - def select_valid_host(namespace, id) - replica_selected = - select_caught_up_replicas(namespace, id) + ::Gitlab::Database::LoadBalancing::Session.current.use_primary! if !result && use_primary_on_failure - ::Gitlab::Database::LoadBalancing::Session.current.use_primary! unless replica_selected + result end # Starts sticking to the primary for the given namespace and id, using # the latest WAL pointer from the primary. def stick(namespace, id) - mark_primary_write_location(namespace, id) + with_primary_write_location do |location| + set_write_location_for(namespace, id, location) + end ::Gitlab::Database::LoadBalancing::Session.current.use_primary! end @@ -102,6 +59,8 @@ module Gitlab ::Gitlab::Database::LoadBalancing::Session.current.use_primary! end + private + def with_primary_write_location # When only using the primary, there's no point in getting write # locations, as the primary is always in sync with itself. @@ -114,12 +73,6 @@ module Gitlab yield(location) end - def mark_primary_write_location(namespace, id) - with_primary_write_location do |location| - set_write_location_for(namespace, id, location) - end - end - def unstick(namespace, id) with_redis do |redis| redis.del(redis_key_for(namespace, id)) @@ -144,8 +97,6 @@ module Gitlab "database-load-balancing/write-location/#{name}/#{namespace}/#{id}" end - private - def with_redis(&block) Gitlab::Redis::DbLoadBalancing.with(&block) end diff --git a/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb b/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb index ef9bee42277..99af167bfc4 100644 --- a/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb +++ b/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb @@ -11,7 +11,7 @@ module Gitlab ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb| if (location = wal_locations.with_indifferent_access[lb.name]) - lb.select_up_to_date_host(location) + lb.select_up_to_date_host(location) != LoadBalancer::NONE_CAUGHT_UP else true end diff --git a/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb b/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb index 8a37e619285..6a82f3813d6 100644 --- a/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb +++ b/lib/gitlab/database/migration_helpers/cascading_namespace_settings.rb @@ -53,7 +53,7 @@ module Gitlab def check_cascading_namespace_setting_consistency(setting_name, lock_name) existing_columns = [] - %w(namespace_settings application_settings).each do |table| + %w[namespace_settings application_settings].each do |table| existing_columns << "#{table}.#{setting_name}" if column_exists?(table.to_sym, setting_name) existing_columns << "#{table}.#{lock_name}" if column_exists?(table.to_sym, lock_name) end diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 5f87bc6bbe2..5a120227d50 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -31,6 +31,8 @@ module Gitlab observation rescue StandardError => error observation.error_message = error.message + + raise ensure observation.walltime = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index 48f58920d52..d6cb9d25728 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -20,19 +20,21 @@ module Gitlab registered_tables.merge(tables) end - def sync_partitions_ignore_db_error - sync_partitions unless ENV['DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP'] + def sync_partitions_ignore_db_error(analyze: false) + sync_partitions(analyze: analyze) unless ENV['DISABLE_POSTGRES_PARTITION_CREATION_ON_STARTUP'] rescue ActiveRecord::ActiveRecordError, PG::Error # ignore - happens when Rake tasks yet have to create a database, e.g. for testing end - def sync_partitions(models_to_sync = registered_for_sync, only_on: nil) + def sync_partitions(models_to_sync = registered_for_sync, only_on: nil, analyze: true) + return if Feature.enabled?(:disallow_database_ddl_feature_flags, type: :ops) + return unless Feature.enabled?(:partition_manager_sync_partitions, type: :ops) Gitlab::AppLogger.info(message: 'Syncing dynamic postgres partitions') Gitlab::Database::EachDatabase.each_model_connection(models_to_sync, only_on: only_on) do |model| - PartitionManager.new(model).sync_partitions + PartitionManager.new(model).sync_partitions(analyze: analyze) end unless only_on @@ -42,7 +44,7 @@ module Gitlab model_connection_name = model.connection_db_config.name Gitlab::Database::EachDatabase.each_connection(include_shared: false) do |connection, connection_name| if connection_name != model_connection_name - PartitionManager.new(model, connection: connection).sync_partitions + PartitionManager.new(model, connection: connection).sync_partitions(analyze: analyze) end end end @@ -60,6 +62,8 @@ module Gitlab end def drop_detached_partitions + return if Feature.enabled?(:disallow_database_ddl_feature_flags, type: :ops) + return unless Feature.enabled?(:partition_manager_sync_partitions, type: :ops) Gitlab::AppLogger.info(message: 'Dropping detached postgres partitions') diff --git a/lib/gitlab/database/partitioning/monthly_strategy.rb b/lib/gitlab/database/partitioning/monthly_strategy.rb index 0f08a47d754..e6af4ac4574 100644 --- a/lib/gitlab/database/partitioning/monthly_strategy.rb +++ b/lib/gitlab/database/partitioning/monthly_strategy.rb @@ -4,18 +4,21 @@ module Gitlab module Database module Partitioning class MonthlyStrategy - attr_reader :model, :partitioning_key, :retain_for, :retain_non_empty_partitions + attr_reader :model, :partitioning_key, :retain_for, :retain_non_empty_partitions, :analyze_interval # We create this many partitions in the future HEADROOM = 6.months delegate :table_name, to: :model - def initialize(model, partitioning_key, retain_for: nil, retain_non_empty_partitions: false) + def initialize( + model, partitioning_key, retain_for: nil, retain_non_empty_partitions: false, + analyze_interval: nil) @model = model @partitioning_key = partitioning_key @retain_for = retain_for @retain_non_empty_partitions = retain_non_empty_partitions + @analyze_interval = analyze_interval end def current_partitions diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index 124fae582d3..cc5c49cc24a 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -4,9 +4,12 @@ module Gitlab module Database module Partitioning class PartitionManager + include ::Gitlab::Utils::StrongMemoize + UnsafeToDetachPartitionError = Class.new(StandardError) - LEASE_TIMEOUT = 1.minute + LEASE_TIMEOUT = 1.hour + STATEMENT_TIMEOUT = 1.hour MANAGEMENT_LEASE_KEY = 'database_partition_management_%s' RETAIN_DETACHED_PARTITIONS_FOR = 1.week @@ -16,7 +19,7 @@ module Gitlab @connection_name = @connection.pool.db_config.name end - def sync_partitions + def sync_partitions(analyze: true) return skip_synching_partitions unless table_partitioned? Gitlab::AppLogger.info( @@ -33,6 +36,8 @@ module Gitlab create(partitions_to_create) unless partitions_to_create.empty? detach(partitions_to_detach) unless partitions_to_detach.empty? + + run_analyze_on_partitioned_table if analyze end rescue ArgumentError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) @@ -146,6 +151,61 @@ module Gitlab connection_name: @connection_name ) end + + def run_analyze_on_partitioned_table + return if Feature.disabled?(:database_analyze_on_partitioned_tables) + return if ineligible_for_analyzing? + + primary_transaction(statement_timeout: STATEMENT_TIMEOUT) do + # Running ANALYZE on partitioned table will go through itself and its partitions + connection.execute("ANALYZE VERBOSE #{model.quoted_table_name}") + end + end + + def ineligible_for_analyzing? + analyze_interval.blank? || + first_model_partition.blank? || + last_analyzed_at_within_interval? + end + + def last_analyzed_at_within_interval? + table_to_query = first_model_partition.identifier + + primary_transaction do + # We don't need to get the last_analyze_time from partitioned table, + # because it's not supported and always returns NULL for PG version below 14 + # Therefore, we can always get the last_analyze_time from the first partition + last_analyzed_at = connection.select_value( + "SELECT pg_stat_get_last_analyze_time('#{table_to_query}'::regclass)" + ) + last_analyzed_at.present? && last_analyzed_at >= Time.current - analyze_interval + end + end + + def first_model_partition + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PostgresPartition.for_parent_table(model.table_name).first + end + end + strong_memoize_attr :first_model_partition + + def analyze_interval + model.partitioning_strategy.analyze_interval + end + + def primary_transaction(statement_timeout: nil) + Gitlab::Database::LoadBalancing::Session.current.use_primary do + connection.transaction(requires_new: false) do + if statement_timeout.present? + connection.execute( + format("SET LOCAL statement_timeout TO '%ds'", statement_timeout) + ) + end + + yield + end + end + end end end end diff --git a/lib/gitlab/database/partitioning/sliding_list_strategy.rb b/lib/gitlab/database/partitioning/sliding_list_strategy.rb index 8f8afdfc551..77997e93480 100644 --- a/lib/gitlab/database/partitioning/sliding_list_strategy.rb +++ b/lib/gitlab/database/partitioning/sliding_list_strategy.rb @@ -4,15 +4,16 @@ module Gitlab module Database module Partitioning class SlidingListStrategy - attr_reader :model, :partitioning_key, :next_partition_if, :detach_partition_if + attr_reader :model, :partitioning_key, :next_partition_if, :detach_partition_if, :analyze_interval delegate :table_name, to: :model - def initialize(model, partitioning_key, next_partition_if:, detach_partition_if:) + def initialize(model, partitioning_key, next_partition_if:, detach_partition_if:, analyze_interval: nil) @model = model @partitioning_key = partitioning_key @next_partition_if = next_partition_if @detach_partition_if = detach_partition_if + @analyze_interval = analyze_interval ensure_partitioning_column_ignored_or_readonly! end diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb index 4e973efebca..c8f6b2ada63 100644 --- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb +++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb @@ -137,7 +137,7 @@ module Gitlab # AND %{column} IS NOT NULL def bucketed_data_sql <<~SQL - WITH hashed_attributes AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (%{source_query}) + WITH hashed_attributes AS MATERIALIZED (%{source_query}) SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num, (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash FROM hashed_attributes diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index b5a45eed3a4..a9f2b963340 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -202,6 +202,7 @@ module Gitlab Rails.env.test? && caller_locations.any? do |l| l.path.end_with?('lib/factory_bot/evaluation.rb') && l.label == 'create' || l.path.end_with?('lib/factory_bot/strategy/create.rb') || + l.path.end_with?('lib/factory_bot/strategy/build.rb') || l.path.end_with?('shoulda/matchers/active_record/validate_uniqueness_of_matcher.rb') && l.label == 'create_existing_record' end end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 4a1b0be848e..c7c93e6f292 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -6,7 +6,7 @@ module Gitlab # Number of indexes to reindex per invocation DEFAULT_INDEXES_PER_INVOCATION = 2 - SUPPORTED_TYPES = %w(btree gist).freeze + SUPPORTED_TYPES = %w[btree gist].freeze # When dropping an index, we acquire a SHARE UPDATE EXCLUSIVE lock, # which only conflicts with DDL and vacuum. We therefore execute this with a rather @@ -16,6 +16,8 @@ module Gitlab REMOVE_INDEX_RETRY_CONFIG = [[1.minute, 9.minutes]] * 30 def self.enabled? + return false if Feature.enabled?(:disallow_database_ddl_feature_flags, type: :ops) + Feature.enabled?(:database_reindexing, type: :ops) end @@ -26,9 +28,15 @@ module Gitlab Gitlab::Database::SharedModel.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) # Hack: Before we do actual reindexing work, create async indexes - Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) + if Feature.disabled?(:disallow_database_ddl_feature_flags, type: :ops) && Feature.enabled?(:database_async_index_creation, type: :ops) + Gitlab::Database::AsyncIndexes.create_pending_indexes! + end + Gitlab::Database::AsyncIndexes.drop_pending_indexes! - Gitlab::Database::AsyncConstraints.validate_pending_entries! if Feature.enabled?(:database_async_foreign_key_validation, type: :ops) + + if Feature.disabled?(:disallow_database_ddl_feature_flags, type: :ops) && Feature.enabled?(:database_async_foreign_key_validation, type: :ops) + Gitlab::Database::AsyncConstraints.validate_pending_entries! + end automatic_reindexing end diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index 7ae1981fa2b..f91146fff3d 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -5,7 +5,7 @@ module Gitlab class TablesTruncate GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo gitlab_embedding].freeze - def initialize(database_name:, min_batch_size:, logger: nil, until_table: nil, dry_run: false) + def initialize(database_name:, min_batch_size: 5, logger: nil, until_table: nil, dry_run: false) @database_name = database_name @min_batch_size = min_batch_size @logger = logger @@ -19,19 +19,6 @@ module Gitlab logger&.info "DRY RUN:" if dry_run - schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) - tables_to_truncate = Gitlab::Database::GitlabSchema.tables_to_schema.reject do |_, schema_name| - GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection).include?(schema_name) - end.keys - - Gitlab::Database::SharedModel.using_connection(connection) do - Postgresql::DetachedPartition.find_each do |detached_partition| - next if GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection).include?(detached_partition.table_schema) - - tables_to_truncate << detached_partition.fully_qualified_table_name - end - end - tables_sorted = Gitlab::Database::TablesSortedByForeignKeys.new(connection, tables_to_truncate).execute # Checking if all the tables have the write-lock triggers # to make sure we are deleting the right tables on the right database. @@ -63,10 +50,41 @@ module Gitlab truncate_tables_in_batches(tables_sorted) end + def needs_truncation? + return false if single_database_setup? + + sql = tables_to_truncate.map { |table_name| "(SELECT EXISTS( SELECT * FROM #{table_name} ))" }.join("\nUNION\n") + + result = with_suppressed_query_analyzers do + connection.execute(sql).to_a + end + + result.to_a.any? { |row| row['exists'] == true } + end + private attr_accessor :database_name, :min_batch_size, :logger, :dry_run, :until_table + def tables_to_truncate + @tables_to_truncate ||= begin + schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) + tables = Gitlab::Database::GitlabSchema.tables_to_schema.reject do |_, schema_name| + GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection).include?(schema_name) + end.keys + + Gitlab::Database::SharedModel.using_connection(connection) do + Postgresql::DetachedPartition.find_each do |detached_partition| + next if GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection).include?(detached_partition.table_schema) + + tables << detached_partition.fully_qualified_table_name + end + end + + tables + end + end + def connection @connection ||= Gitlab::Database.database_base_models[database_name].connection end @@ -133,6 +151,12 @@ module Gitlab ci_base_model = Gitlab::Database.database_base_models[:ci] !!Gitlab::Database.db_config_share_with(ci_base_model.connection_db_config) end + + def with_suppressed_query_analyzers(&block) + Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection.with_suppressed do + Gitlab::Database::QueryAnalyzers::Ci::PartitioningRoutingAnalyzer.with_suppressed(&block) + end + end end end end diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb index 990fd53a370..000a1f50a92 100644 --- a/lib/gitlab/database_importers/work_items/base_type_importer.rb +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -20,7 +20,7 @@ module Gitlab requirement_legacy: 'Requirement legacy', test_reports: 'Test reports', notifications: 'Notifications', - current_user_todos: "Current user todos", + current_user_todos: 'Current user todos', award_emoji: 'Award emoji', linked_items: 'Linked items' }.freeze @@ -124,6 +124,7 @@ module Gitlab :health_status, :status, :notifications, + :current_user_todos, :award_emoji, :linked_items ], @@ -140,7 +141,8 @@ module Gitlab :health_status, :notifications, :current_user_todos, - :award_emoji + :award_emoji, + :linked_items ] }.freeze diff --git a/lib/gitlab/database_warnings.rb b/lib/gitlab/database_warnings.rb new file mode 100644 index 00000000000..8df60ac6404 --- /dev/null +++ b/lib/gitlab/database_warnings.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseWarnings + def self.check_postgres_version_and_print_warning + return if Gitlab::Runtime.rails_runner? + + Gitlab::Database.database_base_models.each do |name, model| + database = Gitlab::Database::Reflection.new(model) + + next if database.postgresql_minimum_supported_version? + + Kernel.warn ERB.new(Rainbow.new.wrap(<<~WARNING).red).result + + ██ ██ █████ ██████ ███ ██ ██ ███ ██ ██████ + ██ ██ ██ ██ ██ ██ ████ ██ ██ ████ ██ ██ + ██ █ ██ ███████ ██████ ██ ██ ██ ██ ██ ██ ██ ██ ███ + ██ ███ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ + ███ ███ ██ ██ ██ ██ ██ ████ ██ ██ ████ ██████ + + ****************************************************************************** + You are using PostgreSQL #{database.version} for the #{name} database, but this version of GitLab requires PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %>. + <% if Rails.env.development? || Rails.env.test? %> + If using gitlab-development-kit, please find the relevant steps here: + https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#upgrade-postgresql + <% end %> + Please upgrade your environment to a supported PostgreSQL version. See + https://docs.gitlab.com/ee/install/requirements.html#database for details. + ****************************************************************************** + WARNING + rescue ActiveRecord::ActiveRecordError, PG::Error + # ignore - happens when Rake tasks yet have to create a database, e.g. for testing + end + end + + def self.check_single_connection_and_print_warning + return if Gitlab::Runtime.rails_runner? + return unless Gitlab::Database.database_mode == Gitlab::Database::MODE_SINGLE_DATABASE + + Kernel.warn ERB.new(Rainbow.new.wrap(<<~WARNING).red).result + + ██ ██ █████ ██████ ███ ██ ██ ███ ██ ██████ + ██ ██ ██ ██ ██ ██ ████ ██ ██ ████ ██ ██ + ██ █ ██ ███████ ██████ ██ ██ ██ ██ ██ ██ ██ ██ ███ + ██ ███ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ ██ + ███ ███ ██ ██ ██ ██ ██ ████ ██ ██ ████ ██████ + + ****************************************************************************** + Your database has a single connection, and single connections were + deprecated in GitLab 15.9 https://docs.gitlab.com/ee/update/deprecations.html#single-database-connection-is-deprecated. + + Please add a :ci section to your database, following these instructions: + https://docs.gitlab.com/ee/install/installation.html#configure-gitlab-db-settings. + ****************************************************************************** + WARNING + end + end +end + +Gitlab::DatabaseWarnings.prepend_mod_with('Gitlab::DatabaseWarnings') diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index aba6e0f033a..ff17010c11c 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -62,9 +62,9 @@ module Gitlab end def link_tag(name, url) - href_attribute = %{href="#{ERB::Util.html_escape_once(url)}" } if Gitlab::UrlSanitizer.valid_web?(url) + href_attribute = %(href="#{ERB::Util.html_escape_once(url)}" ) if Gitlab::UrlSanitizer.valid_web?(url) - %{<a #{href_attribute}rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}.html_safe + %(<a #{href_attribute}rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>).html_safe end # Links package names based on regex. diff --git a/lib/gitlab/diff/char_diff.rb b/lib/gitlab/diff/char_diff.rb index 1b3af8f75ca..d686c43185b 100644 --- a/lib/gitlab/diff/char_diff.rb +++ b/lib/gitlab/diff/char_diff.rb @@ -47,7 +47,7 @@ module Gitlab def to_html @changes.map do |op, text| - %{<span class="#{html_class_names(op)}">#{ERB::Util.html_escape(text)}</span>} + %(<span class="#{html_class_names(op)}">#{ERB::Util.html_escape(text)}</span>) end.join.html_safe end diff --git a/lib/gitlab/diff/inline_diff_marker.rb b/lib/gitlab/diff/inline_diff_marker.rb index c8cc1c0e649..f5afe189697 100644 --- a/lib/gitlab/diff/inline_diff_marker.rb +++ b/lib/gitlab/diff/inline_diff_marker.rb @@ -9,7 +9,7 @@ module Gitlab def mark(line_inline_diffs) super(line_inline_diffs) do |text, left:, right:, mode:| - %{<span class="#{html_class_names(left, right, mode)}">#{text}</span>}.html_safe + %(<span class="#{html_class_names(left, right, mode)}">#{text}</span>).html_safe end end diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 75127098600..bb138115c7d 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -6,7 +6,7 @@ module Gitlab # When SERIALIZE_KEYS is updated, to reset the redis cache entries you'll # need to bump the VERSION constant on Gitlab::Diff::HighlightCache # - SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze + SERIALIZE_KEYS = %i[line_code rich_text text type index old_pos new_pos].freeze attr_reader :marker_ranges attr_writer :text, :rich_text diff --git a/lib/gitlab/doorkeeper_secret_storing/token/unique_application_token.rb b/lib/gitlab/doorkeeper_secret_storing/token/unique_application_token.rb new file mode 100644 index 00000000000..e2ad2544b6c --- /dev/null +++ b/lib/gitlab/doorkeeper_secret_storing/token/unique_application_token.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module DoorkeeperSecretStoring + module Token + class UniqueApplicationToken + # Acronym for 'GitLab OAuth Application Secret' + OAUTH_APPLICATION_SECRET_PREFIX_FORMAT = "gloas-%{token}" + + # Maintains compatibility with ::Doorkeeper::OAuth::Helpers::UniqueToken + # Returns a secure random token, prefixed with a GitLab identifier. + def self.generate(*) + format(OAUTH_APPLICATION_SECRET_PREFIX_FORMAT, token: SecureRandom.hex(32)) + end + end + end + end +end diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index e6c64e2b1d6..7daa1bb96a1 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -50,7 +50,7 @@ module Gitlab end def create_note - external_author = from_address if author == User.support_bot + external_author = from_address if author == Users::Internal.support_bot sent_notification.create_reply(note_message, external_author) end @@ -67,7 +67,7 @@ module Gitlab def validate_from_address! # Recipieint is always set to Support bot for ServiceDesk issues so we should exclude those. - return if author == User.support_bot + return if author == Users::Internal.support_bot raise UserNotFoundError unless from_address && author.verified_email?(from_address) end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 5d0e6ea61e1..949fa554aeb 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -117,7 +117,7 @@ module Gitlab def create_issue! result = ::Issues::CreateService.new( container: project, - current_user: User.support_bot, + current_user: Users::Internal.support_bot, params: { title: mail.subject, description: message_including_template, @@ -199,7 +199,7 @@ module Gitlab def create_note(note) ::Notes::CreateService.new( project, - User.support_bot, + Users::Internal.support_bot, noteable: @issue, note: note ).execute @@ -214,7 +214,7 @@ module Gitlab end def author - User.support_bot + Users::Internal.support_bot end def add_email_participant diff --git a/lib/gitlab/email/message/in_product_marketing.rb b/lib/gitlab/email/message/in_product_marketing.rb deleted file mode 100644 index bd2c91755c8..00000000000 --- a/lib/gitlab/email/message/in_product_marketing.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - UnknownTrackError = Class.new(StandardError) - - def self.for(track) - valid_tracks = Namespaces::InProductMarketingEmailsService::TRACKS.keys - raise UnknownTrackError unless valid_tracks.include?(track) - - "Gitlab::Email::Message::InProductMarketing::#{track.to_s.classify}".constantize - end - end - end - end -end diff --git a/lib/gitlab/email/message/in_product_marketing/admin_verify.rb b/lib/gitlab/email/message/in_product_marketing/admin_verify.rb deleted file mode 100644 index 888f84cde23..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/admin_verify.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - class AdminVerify < Base - def subject_line - s_('InProductMarketing|Create a custom CI runner with just a few clicks') - end - - def tagline - nil - end - - def title - s_('InProductMarketing|Spin up an autoscaling runner in GitLab') - end - - def subtitle - s_('InProductMarketing|Use our AWS cloudformation template to spin up your runners in just a few clicks!') - end - - def body_line1 - '' - end - - def body_line2 - '' - end - - def cta_text - s_('InProductMarketing|Create a custom runner') - end - - def progress - super(track_name: 'Admin') - end - - def invite_members? - user.can?(:admin_group_member, group) - end - end - end - end - end -end diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb deleted file mode 100644 index bd20b7e5fc7..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/base.rb +++ /dev/null @@ -1,127 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - class Base - include Gitlab::Email::Message::InProductMarketing::Helper - include Gitlab::Routing - include Gitlab::Experiment::Dsl - - attr_accessor :format - - def initialize(group:, user:, series:, format: :html) - @series = series - @group = group - @user = user - @format = format - - validate_series! - end - - def subject_line - raise NotImplementedError - end - - def tagline - raise NotImplementedError - end - - def title - raise NotImplementedError - end - - def subtitle - raise NotImplementedError - end - - def body_line1 - raise NotImplementedError - end - - def body_line2 - raise NotImplementedError - end - - def cta_text - raise NotImplementedError - end - - def cta_link - case format - when :html - ActionController::Base.helpers.link_to cta_text, group_email_campaigns_url(group, track: track, series: series), target: '_blank', rel: 'noopener noreferrer' - else - [cta_text, group_email_campaigns_url(group, track: track, series: series)].join(' >> ') - end - end - - def invite_members? - false - end - - def invite_text - s_('InProductMarketing|Do you have a teammate who would be perfect for this task?') - end - - def invite_link - action_link(s_('InProductMarketing|Invite them to help out.'), group_url(group, open_modal: 'invite_members_for_task')) - end - - def unsubscribe - self_managed_preferences_link = marketing_preference_link(track, series) - unsubscribe_message(self_managed_preferences_link) - end - - def progress(current: series + 1, total: total_series, track_name: track.to_s.humanize) - if Gitlab.com? - s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series.') % { current_series: current, total_series: total, track: track_name } - else - s_('InProductMarketing|This is email %{current_series} of %{total_series} in the %{track} series. To disable notification emails sent by your local GitLab instance, either contact your administrator or %{unsubscribe_link}.') % { current_series: current, total_series: total, track: track_name, unsubscribe_link: unsubscribe_link } - end - end - - def logo_path - ["mailers/in_product_marketing", "#{track}-#{series}.png"].join('/') - end - - def series? - total_series > 0 - end - - protected - - attr_reader :group, :user, :series - - private - - def track - self.class.name.demodulize.underscore.to_sym - end - - def total_series - Namespaces::InProductMarketingEmailsService::TRACKS[track][:interval_days].size - end - - def marketing_preference_link(track, series) - params = { - utm_source: 'SM', - utm_medium: 'email', - utm_campaign: 'onboarding', - utm_term: "#{track}_#{series}" - } - - preference_link = "https://about.gitlab.com/company/preference-center/?#{params.to_query}" - - link(s_('InProductMarketing|update your preferences'), preference_link) - end - - def validate_series! - raise ArgumentError, "Only #{total_series} series available for this track." unless @series.between?(0, total_series - 1) - end - end - end - end - end -end diff --git a/lib/gitlab/email/message/in_product_marketing/create.rb b/lib/gitlab/email/message/in_product_marketing/create.rb deleted file mode 100644 index 68f9a9a21c9..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/create.rb +++ /dev/null @@ -1,105 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - class Create < Base - def subject_line - [ - s_('InProductMarketing|Create a project in GitLab in 5 minutes'), - s_('InProductMarketing|Import your project and code from GitHub, Bitbucket and others'), - s_('InProductMarketing|Understand repository mirroring') - ][series] - end - - def tagline - [ - s_('InProductMarketing|Get started today'), - s_('InProductMarketing|Get our import guides'), - s_('InProductMarketing|Need an alternative to importing?') - ][series] - end - - def title - [ - s_('InProductMarketing|Take your first steps with GitLab'), - s_('InProductMarketing|Start by importing your projects'), - s_('InProductMarketing|How (and why) mirroring makes sense') - ][series] - end - - def subtitle - [ - s_('InProductMarketing|Dig in and create a project and a repo'), - s_("InProductMarketing|Here's what you need to know"), - s_('InProductMarketing|Try it out') - ][series] - end - - def body_line1 - [ - s_("InProductMarketing|To understand and get the most out of GitLab, start at the beginning and %{project_link}. In GitLab, repositories are part of a project, so after you've created your project you can go ahead and %{repo_link}.") % { project_link: project_link, repo_link: repo_link }, - s_("InProductMarketing|Making the switch? It's easier than you think to import your projects into GitLab. Move %{github_link}, or import something %{bitbucket_link}.") % { github_link: github_link, bitbucket_link: bitbucket_link }, - s_("InProductMarketing|Sometimes you're not ready to make a full transition to a new tool. If you're not ready to fully commit, %{mirroring_link} gives you a safe way to try out GitLab in parallel with your current tool.") % { mirroring_link: mirroring_link } - ][series] - end - - def body_line2 - [ - s_("InProductMarketing|That's all it takes to get going with GitLab, but if you're new to working with Git, check out our %{basics_link} for helpful tips and tricks for getting started.") % { basics_link: basics_link }, - s_("InProductMarketing|Have a different instance you'd like to import? Here's our %{import_link}.") % { import_link: import_link }, - s_("InProductMarketing|It's also possible to simply %{external_repo_link} in order to take advantage of GitLab's CI/CD.") % { external_repo_link: external_repo_link } - ][series] - end - - def cta_text - [ - s_('InProductMarketing|Create your first project!'), - s_('InProductMarketing|Master the art of importing!'), - s_('InProductMarketing|Understand your project options') - ][series] - end - - def invite_members? - user.can?(:admin_group_member, group) - end - - private - - def project_link - link(s_('InProductMarketing|create a project'), help_page_url('user/project/index')) - end - - def repo_link - link(s_('InProductMarketing|set up a repo'), help_page_url('user/project/repository/index', anchor: 'create-a-repository')) - end - - def github_link - link(s_('InProductMarketing|GitHub Enterprise projects to GitLab'), help_page_url('user/project/import/github')) - end - - def bitbucket_link - link(s_('InProductMarketing|from Bitbucket'), help_page_url('user/project/import/bitbucket_server')) - end - - def mirroring_link - link(s_('InProductMarketing|repository mirroring'), help_page_url('user/project/repository/mirror/index')) - end - - def basics_link - link(s_('InProductMarketing|Git basics'), help_page_url('topics/git/index')) - end - - def import_link - link(s_('InProductMarketing|comprehensive guide'), help_page_url('user/project/import/index')) - end - - def external_repo_link - link(s_('InProductMarketing|connect an external repository'), new_project_url(anchor: 'cicd_for_external_repo')) - end - end - end - end - end -end diff --git a/lib/gitlab/email/message/in_product_marketing/helper.rb b/lib/gitlab/email/message/in_product_marketing/helper.rb index 73d1e0743cc..0770e5f4d76 100644 --- a/lib/gitlab/email/message/in_product_marketing/helper.rb +++ b/lib/gitlab/email/message/in_product_marketing/helper.rb @@ -64,15 +64,6 @@ module Gitlab ] end - def list(array) - case format - when :html - tag.ul { array.map { |item| tag.li item } } - else - '- ' + array.join("\n- ") - end - end - def strong_options case format when :html diff --git a/lib/gitlab/email/message/in_product_marketing/team.rb b/lib/gitlab/email/message/in_product_marketing/team.rb deleted file mode 100644 index ca99dd12c8e..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/team.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - class Team < Base - def subject_line - [ - s_('InProductMarketing|Working in GitLab = more efficient'), - s_("InProductMarketing|Multiple owners, confusing workstreams? We've got you covered"), - s_('InProductMarketing|Your teams can be more efficient') - ][series] - end - - def tagline - [ - s_('InProductMarketing|Invite your colleagues to join in less than one minute'), - s_('InProductMarketing|Get your team set up on GitLab'), - nil - ][series] - end - - def title - [ - s_('InProductMarketing|Team work makes the dream work'), - s_('InProductMarketing|*GitLab*, noun: a synonym for efficient teams'), - s_('InProductMarketing|Find out how your teams are really doing') - ][series] - end - - def subtitle - [ - s_('InProductMarketing|Actually, GitLab makes the team work (better)'), - s_('InProductMarketing|Our tool brings all the things together'), - s_("InProductMarketing|It's all in the stats") - ][series] - end - - def body_line1 - [ - [ - s_('InProductMarketing|Did you know teams that use GitLab are far more efficient?'), - list([ - s_('InProductMarketing|Goldman Sachs went from 1 build every two weeks to thousands of builds a day'), - s_('InProductMarketing|Ticketmaster decreased their CI build time by 15X') - ]) - ].join("\n"), - s_("InProductMarketing|We know a thing or two about efficiency and we don't want to keep that to ourselves. Sign up for a free trial of GitLab Ultimate and your teams will be on it from day one."), - [ - s_('InProductMarketing|Stop wondering and use GitLab to answer questions like:'), - list([ - s_('InProductMarketing|How long does it take us to close issues/MRs by types like feature requests, bugs, tech debt, security?'), - s_('InProductMarketing|How many days does it take our team to complete various tasks?'), - s_('InProductMarketing|What does our value stream timeline look like from product to development to review and production?') - ]) - ].join("\n") - ][series] - end - - def body_line2 - [ - s_('InProductMarketing|Invite your colleagues and start shipping code faster.'), - s_("InProductMarketing|Streamline code review, know at a glance who's unavailable, communicate in comments or in email and integrate with Slack so everyone's on the same page."), - s_('InProductMarketing|When your team is on GitLab these answers are a click away.') - ][series] - end - - def cta_text - [ - s_('InProductMarketing|Invite your colleagues today'), - s_('InProductMarketing|Invite your team in less than 60 seconds'), - s_('InProductMarketing|Invite your team now') - ][series] - end - - def progress - super(current: series + 2, total: 4) - end - end - end - end - end -end diff --git a/lib/gitlab/email/message/in_product_marketing/team_short.rb b/lib/gitlab/email/message/in_product_marketing/team_short.rb deleted file mode 100644 index 1d60a5fe4e5..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/team_short.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - class TeamShort < Base - def subject_line - s_('InProductMarketing|Team up in GitLab for greater efficiency') - end - - def tagline - nil - end - - def title - s_('InProductMarketing|Turn coworkers into collaborators') - end - - def subtitle - s_('InProductMarketing|Invite your team today to build better code (and processes) together') - end - - def body_line1 - '' - end - - def body_line2 - '' - end - - def cta_text - s_('InProductMarketing|Invite your colleagues today') - end - - def progress - super(total: 4, track_name: 'Team') - end - - def logo_path - 'mailers/in_product_marketing/team-0.png' - end - end - end - end - end -end diff --git a/lib/gitlab/email/message/in_product_marketing/trial.rb b/lib/gitlab/email/message/in_product_marketing/trial.rb deleted file mode 100644 index 720262816b4..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/trial.rb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - class Trial < Base - def subject_line - [ - s_('InProductMarketing|Go farther with GitLab'), - s_('InProductMarketing|Automated security scans directly within GitLab'), - s_('InProductMarketing|Take your source code management to the next level') - ][series] - end - - def tagline - [ - s_('InProductMarketing|Start a free trial of GitLab Ultimate – no credit card required'), - s_('InProductMarketing|Improve app security with a 30-day trial'), - s_('InProductMarketing|Start with a GitLab Ultimate free trial') - ][series] - end - - def title - [ - s_('InProductMarketing|Give us one minute...'), - s_("InProductMarketing|Security that's integrated into your development lifecycle"), - s_('InProductMarketing|Improve code quality and streamline reviews') - ][series] - end - - def subtitle - [ - s_('InProductMarketing|...and you can get a free trial of GitLab Ultimate'), - s_('InProductMarketing|Try GitLab Ultimate for free'), - s_('InProductMarketing|Better code in less time') - ][series] - end - - def body_line1 - [ - [ - s_("InProductMarketing|GitLab's premium tiers are designed to make you, your team and your application more efficient and more secure with features including but not limited to:"), - list([ - s_('InProductMarketing|%{strong_start}Company wide portfolio management%{strong_end} — including multi-level epics, scoped labels').html_safe % strong_options, - s_('InProductMarketing|%{strong_start}Multiple approval roles%{strong_end} — including code owners and required merge approvals').html_safe % strong_options, - s_('InProductMarketing|%{strong_start}Advanced application security%{strong_end} — including SAST, DAST scanning, FUZZ testing, dependency scanning, license compliance, secrete detection').html_safe % strong_options, - s_('InProductMarketing|%{strong_start}Executive level insights%{strong_end} — including reporting on productivity, tasks by type, days to completion, value stream').html_safe % strong_options - ]) - ].join("\n"), - s_('InProductMarketing|GitLab provides static application security testing (SAST), dynamic application security testing (DAST), container scanning, and dependency scanning to help you deliver secure applications along with license compliance.'), - s_('InProductMarketing|By enabling code owners and required merge approvals the right person will review the right MR. This is a win-win: cleaner code and a more efficient review process.') - ][series] - end - - def body_line2 - [ - s_('InProductMarketing|Start a GitLab Ultimate trial today in less than one minute, no credit card required.'), - s_('InProductMarketing|Get started today with a 30-day GitLab Ultimate trial, no credit card required.'), - s_('InProductMarketing|Code owners and required merge approvals are part of the paid tiers of GitLab. You can start a free 30-day trial of GitLab Ultimate and enable these features in less than 5 minutes with no credit card required.') - ][series] - end - - def cta_text - [ - s_('InProductMarketing|Start a trial'), - s_('InProductMarketing|Beef up your security'), - s_('InProductMarketing|Start your trial now!') - ][series] - end - - def progress - super(current: series + 2, total: 4) - end - end - end - end - end -end diff --git a/lib/gitlab/email/message/in_product_marketing/trial_short.rb b/lib/gitlab/email/message/in_product_marketing/trial_short.rb deleted file mode 100644 index 0fcd3fde4a6..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/trial_short.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - class TrialShort < Base - def subject_line - s_('InProductMarketing|Be a DevOps hero') - end - - def tagline - nil - end - - def title - s_('InProductMarketing|Expand your DevOps journey with a free GitLab trial') - end - - def subtitle - s_('InProductMarketing|Start your trial today to experience single application success and discover all the features of GitLab Ultimate for free!') - end - - def body_line1 - '' - end - - def body_line2 - '' - end - - def cta_text - s_('InProductMarketing|Start a trial') - end - - def progress - super(total: 4, track_name: 'Trial') - end - - def logo_path - 'mailers/in_product_marketing/trial-0.png' - end - end - end - end - end -end diff --git a/lib/gitlab/email/message/in_product_marketing/verify.rb b/lib/gitlab/email/message/in_product_marketing/verify.rb deleted file mode 100644 index 3982a8b87fd..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/verify.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - class Verify < Base - def subject_line - [ - s_('InProductMarketing|Feel the need for speed?'), - s_('InProductMarketing|3 ways to dive into GitLab CI/CD'), - s_('InProductMarketing|Explore the power of GitLab CI/CD') - ][series] - end - - def tagline - [ - s_('InProductMarketing|Use GitLab CI/CD'), - s_('InProductMarketing|Test, create, deploy'), - s_('InProductMarketing|Are your runners ready?') - ][series] - end - - def title - [ - s_('InProductMarketing|Rapid development, simplified'), - s_('InProductMarketing|Get started with GitLab CI/CD'), - s_('InProductMarketing|Launch GitLab CI/CD in 20 minutes or less') - ][series] - end - - def subtitle - [ - s_('InProductMarketing|How to build and test faster'), - s_('InProductMarketing|Explore the options'), - s_('InProductMarketing|Follow our steps') - ][series] - end - - def body_line1 - [ - s_("InProductMarketing|Tired of wrestling with disparate tool chains, information silos and inefficient processes? GitLab's CI/CD is built on a DevOps platform with source code management, planning, monitoring and more ready to go. Find out %{ci_link}.") % { ci_link: ci_link }, - s_("InProductMarketing|GitLab's CI/CD makes software development easier. Don't believe us? Here are three ways you can take it for a fast (and satisfying) test drive:"), - s_("InProductMarketing|Get going with CI/CD quickly using our %{quick_start_link}. Start with an available runner and then create a CI .yml file – it's really that easy.") % { quick_start_link: quick_start_link } - ][series] - end - - def body_line2 - [ - nil, - list([ - s_('InProductMarketing|Start by %{performance_link}').html_safe % { performance_link: performance_link }, - s_('InProductMarketing|Move on to easily creating a Pages website %{ci_template_link}').html_safe % { ci_template_link: ci_template_link }, - s_('InProductMarketing|And finally %{deploy_link} a Python application.').html_safe % { deploy_link: deploy_link } - ]), - nil - ][series] - end - - def cta_text - [ - s_('InProductMarketing|Get to know GitLab CI/CD'), - s_('InProductMarketing|Try it yourself'), - s_('InProductMarketing|Explore GitLab CI/CD') - ][series] - end - - def invite_members? - user.can?(:admin_group_member, group) - end - - private - - def ci_link - link(s_('InProductMarketing|how easy it is to get started'), help_page_url('ci/index')) - end - - def quick_start_link - link(s_('InProductMarketing|quick start guide'), help_page_url('ci/quick_start/index')) - end - - def performance_link - link(s_('InProductMarketing|testing browser performance'), help_page_url('user/project/merge_requests/browser_performance_testing')) - end - - def ci_template_link - link(s_('InProductMarketing|using a CI/CD template'), help_page_url('user/project/pages/getting_started/pages_ci_cd_template')) - end - - def deploy_link - link(s_('InProductMarketing|test and deploy'), help_page_url('ci/examples/test-and-deploy-python-application-to-heroku')) - end - end - end - end - end -end diff --git a/lib/gitlab/email/service_desk/custom_email.rb b/lib/gitlab/email/service_desk/custom_email.rb new file mode 100644 index 00000000000..30ae435a6ec --- /dev/null +++ b/lib/gitlab/email/service_desk/custom_email.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module ServiceDesk + # Doesn't include Gitlab::Email::Common because a custom email doesn't + # support all features and methods of ingestable email addresses like + # incoming_email and service_desk_email. + module CustomEmail + class << self + def reply_address(issue, reply_key) + return if reply_key.nil? + + custom_email = issue&.project&.service_desk_setting&.custom_email + return if custom_email.nil? + + # Reply keys for custom email addresses always go before the @. + # We don't have a placeholder. + custom_email.sub('@', "+#{reply_key}@") + end + end + end + end + end +end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 786a68c86f2..5f06f37821c 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -57,7 +57,7 @@ module Gitlab end def weak_etag_format(value) - %{W/"#{value}"} + %(W/"#{value}") end def handle_cache_hit(etag, route, request) diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb index 7946e768e00..50cdde88e2d 100644 --- a/lib/gitlab/etag_caching/router/graphql.rb +++ b/lib/gitlab/etag_caching/router/graphql.rb @@ -9,7 +9,7 @@ module Gitlab ROUTES = [ [ - %r(\Apipelines/id/\d+\z), + %r{\Apipelines/id/\d+\z}, 'pipelines_graph', 'continuous_integration' ], @@ -19,12 +19,12 @@ module Gitlab 'pipeline_composition' ], [ - %r(\Aon_demand_scan/counts/), + %r{\Aon_demand_scan/counts/}, 'on_demand_scans', 'dynamic_application_security_testing' ], [ - %r(\A/projects/.+/-/environments.json\z), + %r{\A/projects/.+/-/environments.json\z}, 'environment_details', 'continuous_delivery' ] diff --git a/lib/gitlab/etag_caching/router/rails.rb b/lib/gitlab/etag_caching/router/rails.rb index 5fd592c43e4..333bffd371f 100644 --- a/lib/gitlab/etag_caching/router/rails.rb +++ b/lib/gitlab/etag_caching/router/rails.rb @@ -10,9 +10,8 @@ module Gitlab # To match a regex the path needs to match the following: # - Don't contain a reserved word (expect for the words used in the # regex itself) - # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route # - Ending in `issues/id`/realtime_changes` for the `issue_title` route - USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes + USED_IN_ROUTES = %w[noteable issue issues realtime_changes commit pipelines merge_requests builds new environments].freeze RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES @@ -21,85 +20,73 @@ module Gitlab ROUTES = [ [ - %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z), - 'issue_notes', - ::Projects::NotesController, - :index - ], - [ - %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z), - 'merge_request_notes', - ::Projects::NotesController, - :index - ], - [ - %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z), + %r{#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z}, 'issue_title', ::Projects::IssuesController, :realtime_changes ], [ - %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z), + %r{#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z}, 'commit_pipelines', ::Projects::CommitController, :pipelines ], [ - %r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z), + %r{#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z}, 'new_merge_request_pipelines', ::Projects::MergeRequests::CreationsController, :new ], [ - %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z), + %r{#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z}, 'merge_request_pipelines', ::Projects::MergeRequestsController, :pipelines ], [ - %r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z), + %r{#{RESERVED_WORDS_PREFIX}/pipelines\.json\z}, 'project_pipelines', ::Projects::PipelinesController, :index ], [ - %r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z), + %r{#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z}, 'project_pipeline', ::Projects::PipelinesController, :show ], [ - %r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z), + %r{#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z}, 'project_build', ::Projects::BuildsController, :show ], [ - %r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z), + %r{#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z}, 'cluster_environments', ::Groups::ClustersController, :environments ], [ - %r(#{RESERVED_WORDS_PREFIX}/-/environments\.json\z), + %r{#{RESERVED_WORDS_PREFIX}/-/environments\.json\z}, 'environments', ::Projects::EnvironmentsController, :index ], [ - %r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z), + %r{#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z}, 'realtime_changes_import_github', ::Import::GithubController, :realtime_changes ], [ - %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), + %r{#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z}, 'realtime_changes_import_gitea', ::Import::GiteaController, :realtime_changes ], [ - %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), + %r{#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z}, 'merge_request_widget', ::Projects::MergeRequests::ContentController, :cached_widget diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index f36a7a0603c..5fdf5ac9436 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -31,9 +31,7 @@ module Gitlab private def with_redis(&blk) - # We use multistore as n interweaving double-write will result in n-1 subsequent requests - # becoming a cache-miss, however, 2 interweaving .touch will lead to 1 cache miss anyway. - Gitlab::Redis::EtagCache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + Gitlab::Redis::Cache.with(&blk) # rubocop:disable CodeReuse/ActiveRecord end def generate_etag diff --git a/lib/gitlab/event_store.rb b/lib/gitlab/event_store.rb index 474aac8073e..1e397b52ddf 100644 --- a/lib/gitlab/event_store.rb +++ b/lib/gitlab/event_store.rb @@ -36,26 +36,6 @@ module Gitlab store.subscribe ::MergeRequests::UpdateHeadPipelineWorker, to: ::Ci::PipelineCreatedEvent store.subscribe ::Namespaces::UpdateRootStatisticsWorker, to: ::Projects::ProjectDeletedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Pages::PageDeployedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Pages::PageDeletedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectDeletedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectCreatedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectPathChangedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectArchivedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectTransferedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, - to: ::Projects::ProjectAttributesChangedEvent, - if: -> (event) { event.pages_related? } - store.subscribe ::Pages::InvalidateDomainCacheWorker, - to: ::Projects::ProjectFeaturesChangedEvent, - if: -> (event) { event.pages_related? } - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Groups::GroupTransferedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Groups::GroupPathChangedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Groups::GroupDeletedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::PagesDomains::PagesDomainDeletedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::PagesDomains::PagesDomainUpdatedEvent - store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::PagesDomains::PagesDomainCreatedEvent - store.subscribe ::MergeRequests::CreateApprovalEventWorker, to: ::MergeRequests::ApprovedEvent store.subscribe ::MergeRequests::CreateApprovalNoteWorker, to: ::MergeRequests::ApprovedEvent store.subscribe ::MergeRequests::ResolveTodosAfterApprovalWorker, to: ::MergeRequests::ApprovedEvent diff --git a/lib/gitlab/event_store/store.rb b/lib/gitlab/event_store/store.rb index 2e5e0215687..318745cc192 100644 --- a/lib/gitlab/event_store/store.rb +++ b/lib/gitlab/event_store/store.rb @@ -15,12 +15,12 @@ module Gitlab lock! end - def subscribe(worker, to:, if: nil) + def subscribe(worker, to:, if: nil, delay: nil) condition = binding.local_variable_get('if') Array(to).each do |event| validate_subscription!(worker, event) - subscriptions[event] << Gitlab::EventStore::Subscription.new(worker, condition) + subscriptions[event] << Gitlab::EventStore::Subscription.new(worker, condition, delay) end end diff --git a/lib/gitlab/event_store/subscription.rb b/lib/gitlab/event_store/subscription.rb index 01986355d2d..81a65f9a8ff 100644 --- a/lib/gitlab/event_store/subscription.rb +++ b/lib/gitlab/event_store/subscription.rb @@ -3,17 +3,22 @@ module Gitlab module EventStore class Subscription - attr_reader :worker, :condition + attr_reader :worker, :condition, :delay - def initialize(worker, condition) + def initialize(worker, condition, delay) @worker = worker @condition = condition + @delay = delay end def consume_event(event) return unless condition_met?(event) - worker.perform_async(event.class.name, event.data.deep_stringify_keys) + if delay + worker.perform_in(delay, event.class.name, event.data.deep_stringify_keys) + else + worker.perform_async(event.class.name, event.data.deep_stringify_keys) + end # We rescue and track any exceptions here because we don't want to # impact other subscribers if one is faulty. diff --git a/lib/gitlab/fips.rb b/lib/gitlab/fips.rb index b2c22182d4b..c71bd0e1ac9 100644 --- a/lib/gitlab/fips.rb +++ b/lib/gitlab/fips.rb @@ -8,15 +8,15 @@ module Gitlab Technology = Gitlab::SSHPublicKey::Technology SSH_KEY_TECHNOLOGIES = [ - Technology.new(:rsa, SSHData::PublicKey::RSA, [3072, 4096], %w(ssh-rsa)), - Technology.new(:dsa, SSHData::PublicKey::DSA, [], %w(ssh-dss)), - Technology.new(:ecdsa, SSHData::PublicKey::ECDSA, [256, 384, 521], %w(ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521)), - Technology.new(:ed25519, SSHData::PublicKey::ED25519, [256], %w(ssh-ed25519)), - Technology.new(:ecdsa_sk, SSHData::PublicKey::SKECDSA, [256], %w(sk-ecdsa-sha2-nistp256@openssh.com)), - Technology.new(:ed25519_sk, SSHData::PublicKey::SKED25519, [256], %w(sk-ssh-ed25519@openssh.com)) + Technology.new(:rsa, SSHData::PublicKey::RSA, [3072, 4096], %w[ssh-rsa]), + Technology.new(:dsa, SSHData::PublicKey::DSA, [], %w[ssh-dss]), + Technology.new(:ecdsa, SSHData::PublicKey::ECDSA, [256, 384, 521], %w[ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521]), + Technology.new(:ed25519, SSHData::PublicKey::ED25519, [256], %w[ssh-ed25519]), + Technology.new(:ecdsa_sk, SSHData::PublicKey::SKECDSA, [256], %w[sk-ecdsa-sha2-nistp256@openssh.com]), + Technology.new(:ed25519_sk, SSHData::PublicKey::SKED25519, [256], %w[sk-ssh-ed25519@openssh.com]) ].freeze - OPENSSL_DIGESTS = %i(SHA1 SHA256 SHA384 SHA512).freeze + OPENSSL_DIGESTS = %i[SHA1 SHA256 SHA384 SHA512].freeze class << self # Returns whether we should be running in FIPS mode or not diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index ca1a2b2a077..b3f4f4fb933 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -115,13 +115,13 @@ module Gitlab labels = [] [bug['sCategory'], bug['sPriority']].each do |label| - unless label.blank? - labels << label + next if label.blank? - unless @known_labels.include?(label) - create_label(label) - @known_labels << label - end + labels << label + + unless @known_labels.include?(label) + create_label(label) + @known_labels << label end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 4e574d6de74..9150213020e 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -21,6 +21,7 @@ module Gitlab InvalidPageToken = Class.new(BaseError) InvalidRefFormatError = Class.new(BaseError) ReferencesLockedError = Class.new(BaseError) + ReferenceStateMismatchError = Class.new(BaseError) class ResourceExhaustedError < BaseError def initialize(msg = nil, retry_after = 0) diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index bb5bbeeb27e..ae90291c0a3 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -120,7 +120,7 @@ module Gitlab end def initialize(options) - %w(id name path size data mode commit_id binary).each do |key| + %w[id name path size data mode commit_id binary].each do |key| self.__send__("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 72f7413500f..de25fa7e099 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -31,9 +31,9 @@ module Gitlab # persisting limits over that. MAX_PATCH_BYTES_UPPER_BOUND = 500.kilobytes - SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze + SERIALIZE_KEYS = %i[diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large].freeze - BINARY_NOTICE_PATTERN = %r(Binary files a\/(.*) and b\/(.*) differ).freeze + BINARY_NOTICE_PATTERN = %r{Binary files a\/(.*) and b\/(.*) differ}.freeze class << self def between(repo, head, base, options = {}, *paths) @@ -81,7 +81,7 @@ module Gitlab # exceeded def filter_diff_options(options, default_options = {}) allowed_options = [:ignore_whitespace_change, :max_files, :max_lines, - :limits, :expanded] + :limits, :expanded, :collect_all_paths] if default_options actual_defaults = default_options.dup @@ -126,6 +126,10 @@ module Gitlab limit / 10 end + def collect_patch_overage? + !!Feature.enabled?(:collect_all_diff_paths) + end + # Returns the limit for a single diff file (patch). # # Patches surpassing this limit shouldn't be persisted in the database @@ -198,9 +202,13 @@ module Gitlab # This is used by `to_hash` and `init_from_hash`. alias_method :too_large, :too_large? - def too_large! + def prune! @diff = '' @line_count = 0 + end + + def too_large! + prune! @too_large = true end @@ -211,11 +219,19 @@ module Gitlab end def collapse! - @diff = '' - @line_count = 0 + prune! @collapsed = true end + def overflow? + return @overflow if defined?(@overflow) + + # If overflow is not defined, we're + # not recieveing a diff from Gitaly + # and overflow has no meaning + false + end + def json_safe_diff return @diff unless detect_binary?(@diff) @@ -248,7 +264,7 @@ module Gitlab end def init_from_gitaly(gitaly_diff) - @diff = gitaly_diff.respond_to?(:patch) ? encode!(gitaly_diff.patch) : '' + @diff = gitaly_diff.try(:patch).present? ? encode!(gitaly_diff.patch) : '' @new_path = encode!(gitaly_diff.to_path.dup) @old_path = encode!(gitaly_diff.from_path.dup) @a_mode = gitaly_diff.old_mode.to_s(8) @@ -257,11 +273,19 @@ module Gitlab @renamed_file = gitaly_diff.from_path != gitaly_diff.to_path @deleted_file = gitaly_diff.to_id == BLANK_SHA @too_large = gitaly_diff.too_large if gitaly_diff.respond_to?(:too_large) + gitaly_overflow = gitaly_diff.try(:overflow_marker) + @overflow = Diff.collect_patch_overage? && gitaly_overflow collapse! if gitaly_diff.respond_to?(:collapsed) && gitaly_diff.collapsed + # Diffs exceeding limits returned from gitaly when "collect_all_paths" are enabled + # are already pruned, but should be "collapsed" as they have no content + @collapsed = true if @overflow end def prune_diff_if_eligible + # If we have overflow, diffs are already pruned, retain line counts + return if overflow? + if too_large? ::Gitlab::Metrics.add_event(:patch_hard_limit_bytes_hit) diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index b4dd880ceb7..c021268a62a 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -13,6 +13,10 @@ module Gitlab { max_files: ::Commit.diff_safe_max_files, max_lines: ::Commit.diff_safe_max_lines } end + def self.collect_all_paths?(collect_all_paths) + Gitlab::Git::Diff.collect_patch_overage? ? collect_all_paths : false + end + def self.limits(options = {}) limits = {} defaults = default_limits @@ -25,7 +29,7 @@ module Gitlab limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes limits[:max_patch_bytes_for_file_extension] = options.fetch(:max_patch_bytes_for_file_extension, {}) - + limits[:collect_all_paths] = collect_all_paths?(options.fetch(:collect_all_paths, false)) limits end @@ -164,7 +168,12 @@ module Gitlab if raw.overflow_marker @overflow = true - break + # If we're requesting patches with `collect_all_paths` enabled, then + # Once we hit the overflow marker, gitlay has still returned diffs, just without + # patches, only metadata + unless @limits[:collect_all_paths] + break + end end yield @array[i] = diff diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d27f721bb2c..dfbf8292f54 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -763,6 +763,20 @@ module Gitlab BatchedGitRefUpdates::Deletion.bulk_insert!(records) end + # Update a list of references from X -> Y + # + # Ref list is expected to be an array of hashes in the form: + # old_sha: + # new_sha + # reference: + # + # When new_sha is Gitlab::Git::BLANK_SHA, then this will be deleted + def update_refs(ref_list) + wrapped_gitaly_errors do + gitaly_ref_client.update_refs(ref_list: ref_list) if ref_list.any? + end + end + def delete_refs(...) wrapped_gitaly_errors do gitaly_delete_refs(...) @@ -959,10 +973,6 @@ module Gitlab gitaly_repository_client.create_from_bundle(bundle_path) end - def create_from_snapshot(url, auth) - gitaly_repository_client.create_from_snapshot(url, auth) - end - def rebase(user, rebase_id, branch:, branch_sha:, remote_repository:, remote_branch:, push_options: [], &block) wrapped_gitaly_errors do gitaly_operation_client.rebase( diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb index ea10b4e7cd8..cd4eefa158e 100644 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ b/lib/gitlab/git/rugged_impl/repository.rb @@ -13,7 +13,7 @@ module Gitlab extend ::Gitlab::Utils::Override include Gitlab::Git::RuggedImpl::UseRugged - FEATURE_FLAGS = %i(rugged_find_commit rugged_tree_entries rugged_tree_entry rugged_commit_is_ancestor rugged_commit_tree_entry rugged_list_commits_by_oid).freeze + FEATURE_FLAGS = %i[rugged_find_commit rugged_tree_entries rugged_tree_entry rugged_commit_is_ancestor rugged_commit_tree_entry rugged_list_commits_by_oid].freeze def alternate_object_directories relative_object_directories.map { |d| File.join(path, d) } diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 0895c0b8a22..6e97e412b91 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -71,7 +71,7 @@ module Gitlab end def initialize(options) - %w(id name path flat_path type mode commit_id).each do |key| + %w[id name path flat_path type mode commit_id].each do |key| self.send("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index a28952ab7bc..45283d51b1b 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -38,8 +38,8 @@ module Gitlab Timing information for debugging purposes: MESSAGE - DOWNLOAD_COMMANDS = %w{git-upload-pack git-upload-archive}.freeze - PUSH_COMMANDS = %w{git-receive-pack}.freeze + DOWNLOAD_COMMANDS = %w[git-upload-pack git-upload-archive].freeze + PUSH_COMMANDS = %w[git-receive-pack].freeze ALL_COMMANDS = DOWNLOAD_COMMANDS + PUSH_COMMANDS attr_reader :actor, :protocol, :authentication_abilities, diff --git a/lib/gitlab/git_access_design.rb b/lib/gitlab/git_access_design.rb index fb8df0d217a..bf89c01305a 100644 --- a/lib/gitlab/git_access_design.rb +++ b/lib/gitlab/git_access_design.rb @@ -4,23 +4,6 @@ module Gitlab class GitAccessDesign < GitAccess extend ::Gitlab::Utils::Override - # TODO Re-factor so that correct container is passed to the constructor - # and this method can be removed from here - # https://gitlab.com/gitlab-org/gitlab/-/issues/409454 - def initialize( - actor, container, protocol, authentication_abilities:, repository_path: nil, redirected_path: nil, - auth_result_type: nil) - super( - actor, - select_container(container), - protocol, - authentication_abilities: authentication_abilities, - repository_path: repository_path, - redirected_path: redirected_path, - auth_result_type: auth_result_type - ) - end - def check(_cmd, _changes) check_protocol! check_can_create_design! @@ -35,10 +18,6 @@ module Gitlab private - def select_container(container) - container.is_a?(::DesignManagement::Repository) ? container.project : container - end - def check_protocol! if protocol != 'web' raise ::Gitlab::GitAccess::ForbiddenError, "Designs are only accessible using the web interface" diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 77d2ba315a8..f2ea6f17d90 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -128,7 +128,7 @@ module Gitlab raise "storage #{storage.inspect} is missing a gitaly_address" end - unless %w(tcp unix tls dns).include?(URI(address).scheme) + unless %w[tcp unix tls dns].include?(URI(address).scheme) raise "Unsupported Gitaly address: #{address.inspect} does not use URL scheme 'tcp' or 'unix' or 'tls' or 'dns'" end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 1ef5b0f96c2..573e3547202 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -418,6 +418,9 @@ module Gitlab response = gitaly_client_call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) response.reduce([]) { |memo, msg| memo << msg.data }.join + # Temporary fix, use structured errors when they are available: https://gitlab.com/gitlab-org/gitaly/-/issues/5594 + rescue GRPC::Internal + "" end def find_commit(revision) diff --git a/lib/gitlab/gitaly_client/diff.rb b/lib/gitlab/gitaly_client/diff.rb index dd192ccde1a..c30b8ab1bc8 100644 --- a/lib/gitlab/gitaly_client/diff.rb +++ b/lib/gitlab/gitaly_client/diff.rb @@ -3,7 +3,7 @@ module Gitlab module GitalyClient class Diff - ATTRS = %i(from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed too_large).freeze + ATTRS = %i[from_path to_path old_mode new_mode from_id to_id patch overflow_marker collapsed too_large].freeze include AttributesBag end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index fe76543548b..905588c2afc 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -553,7 +553,8 @@ module Gitlab message: encode_binary(message), start_branch_name: encode_binary(start_branch_name.to_s), start_repository: start_repository.gitaly_repository, - dry_run: dry_run + dry_run: dry_run, + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) gitaly_client_call( diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 45edfd4cbbf..c251c126b07 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -122,6 +122,37 @@ module Gitlab end end + def update_refs(ref_list:) + request = Enumerator.new do |y| + ref_list.each_slice(100) do |refs| + updates = refs.map do |ref_pair| + Gitaly::UpdateReferencesRequest::Update.new( + old_object_id: ref_pair[:old_sha], + new_object_id: ref_pair[:new_sha], + reference: encode_binary(ref_pair[:reference]) + ) + end + + y.yield Gitaly::UpdateReferencesRequest.new(repository: @gitaly_repo, updates: updates) + end + end + + gitaly_client_call(@repository.storage, :ref_service, :update_references, request, timeout: GitalyClient.long_timeout) + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error.try(:error) + when :invalid_format + raise Gitlab::Git::InvalidRefFormatError, "references have an invalid format: #{detailed_error.invalid_format.refs.join(",")}" + when :references_locked + raise Gitlab::Git::ReferencesLockedError + when :reference_state_mismatch + raise Gitlab::Git::ReferenceStateMismatchError + else + raise e + end + end + def delete_refs(refs: [], except_with_prefixes: []) request = Gitaly::DeleteRefsRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index b2d5f9c7e13..9ea541e083d 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -237,22 +237,6 @@ module Gitlab ) end - def create_from_snapshot(http_url, http_auth) - request = Gitaly::CreateRepositoryFromSnapshotRequest.new( - repository: @gitaly_repo, - http_url: http_url, - http_auth: http_auth - ) - - gitaly_client_call( - @storage, - :repository_service, - :create_repository_from_snapshot, - request, - timeout: GitalyClient.long_timeout - ) - end - def write_ref(ref_path, ref, old_ref) request = Gitaly::WriteRefRequest.new( repository: @gitaly_repo, diff --git a/lib/gitlab/gitaly_client/wiki_page.rb b/lib/gitlab/gitaly_client/wiki_page.rb index 757a429fb8a..f6ea2c96bbb 100644 --- a/lib/gitlab/gitaly_client/wiki_page.rb +++ b/lib/gitlab/gitaly_client/wiki_page.rb @@ -3,7 +3,7 @@ module Gitlab module GitalyClient class WikiPage - ATTRS = %i(title format url_path path name historical raw_data).freeze + ATTRS = %i[title format url_path path name historical raw_data].freeze include AttributesBag include Gitlab::EncodingHelper diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 3b19b9d16d2..d48b25842b3 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -26,7 +26,7 @@ module Gitlab def self.ghost_user_id key = 'github-import/ghost-user-id' - Gitlab::Cache::Import::Caching.read_integer(key) || Gitlab::Cache::Import::Caching.write(key, User.select(:id).ghost.id) + Gitlab::Cache::Import::Caching.read_integer(key) || Gitlab::Cache::Import::Caching.write(key, Users::Internal.ghost.id) end # Get formatted GitHub import URL. If github.com is in the import URL, this will return nil and octokit will use the default github.com API URL diff --git a/lib/gitlab/github_import/attachments_downloader.rb b/lib/gitlab/github_import/attachments_downloader.rb index b71d5f753f2..4db55a6aabb 100644 --- a/lib/gitlab/github_import/attachments_downloader.rb +++ b/lib/gitlab/github_import/attachments_downloader.rb @@ -8,15 +8,17 @@ module Gitlab include ::BulkImports::FileDownloads::Validations DownloadError = Class.new(StandardError) + UnsupportedAttachmentError = Class.new(StandardError) FILENAME_SIZE_LIMIT = 255 # chars before the extension DEFAULT_FILE_SIZE_LIMIT = 25.megabytes TMP_DIR = File.join(Dir.tmpdir, 'github_attachments').freeze - attr_reader :file_url, :filename, :file_size_limit + attr_reader :file_url, :filename, :file_size_limit, :options - def initialize(file_url, file_size_limit: DEFAULT_FILE_SIZE_LIMIT) + def initialize(file_url, options: {}, file_size_limit: DEFAULT_FILE_SIZE_LIMIT) @file_url = file_url + @options = options @file_size_limit = file_size_limit filename = URI(file_url).path.split('/').last @@ -27,7 +29,9 @@ module Gitlab validate_content_length validate_filepath - file = download + redirection_url = get_assets_download_redirection_url + file = download_from(redirection_url) + validate_symlink file end @@ -47,9 +51,34 @@ module Gitlab Gitlab::HTTP.perform_request(Net::HTTP::Head, file_url, {}).headers end - def download + # Github /assets redirection link will redirect to aws which has its own authorization. + # Keeping our bearer token will cause request rejection + # eg. Only one auth mechanism allowed; only the X-Amz-Algorithm query parameter, + # Signature query string parameter or the Authorization header should be specified. + def get_assets_download_redirection_url + return file_url unless file_url.starts_with?(github_assets_url_regex) + + options[:follow_redirects] = false + response = Gitlab::HTTP.perform_request(Net::HTTP::Get, file_url, options) + raise_error("expected a redirect response, got #{response.code}") unless response.redirection? + + redirection_url = response.headers[:location] + filename = URI.parse(redirection_url).path + + unless Gitlab::GithubImport::Markdown::Attachment::MEDIA_TYPES.any? { |type| filename.ends_with?(type) } + raise UnsupportedAttachmentError + end + + redirection_url + end + + def github_assets_url_regex + %r{#{Regexp.escape(::Gitlab::GithubImport::MarkdownText.github_url)}/.*/assets/} + end + + def download_from(url) file = File.open(filepath, 'wb') - Gitlab::HTTP.perform_request(Net::HTTP::Get, file_url, stream_body: true) { |batch| file.write(batch) } + Gitlab::HTTP.perform_request(Net::HTTP::Get, url, stream_body: true) { |batch| file.write(batch) } file end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 886563a6f69..23d4faa3dde 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -67,10 +67,18 @@ module Gitlab end # Returns the details of a GitHub user. + # 304 (Not Modified) status means the user is cached - API won't return user data. # - # username - The username of the user. - def user(username) - with_rate_limit { octokit.user(username).to_h } + # @param username[String] the username of the user. + # @param options[Hash] the optional parameters. + def user(username, options = {}) + with_rate_limit do + user = octokit.user(username, options) + + next if octokit.last_response&.status == 304 + + user.to_h + end end def pull_request_reviews(repo_name, iid) diff --git a/lib/gitlab/github_import/importer/note_attachments_importer.rb b/lib/gitlab/github_import/importer/note_attachments_importer.rb index 266ee2938ba..26472b0d468 100644 --- a/lib/gitlab/github_import/importer/note_attachments_importer.rb +++ b/lib/gitlab/github_import/importer/note_attachments_importer.rb @@ -4,14 +4,15 @@ module Gitlab module GithubImport module Importer class NoteAttachmentsImporter - attr_reader :note_text, :project + attr_reader :note_text, :project, :client # note_text - An instance of `Gitlab::GithubImport::Representation::NoteText`. # project - An instance of `Project`. # client - An instance of `Gitlab::GithubImport::Client`. - def initialize(note_text, project, _client = nil) + def initialize(note_text, project, client) @note_text = note_text @project = project + @client = client end def execute @@ -33,7 +34,7 @@ module Gitlab if attachment.part_of_project_blob?(project_import_source) convert_project_content_link(attachment.url, project_import_source) - elsif attachment.media? || attachment.doc_belongs_to_project?(project_import_source) + elsif attachment.media?(project_import_source) || attachment.doc_belongs_to_project?(project_import_source) download_attachment(attachment) else # url to other GitHub project attachment.url @@ -53,14 +54,24 @@ module Gitlab # in: an instance of Gitlab::GithubImport::Markdown::Attachment # out: gitlab attachment markdown url def download_attachment(attachment) - downloader = ::Gitlab::GithubImport::AttachmentsDownloader.new(attachment.url) + downloader = ::Gitlab::GithubImport::AttachmentsDownloader.new(attachment.url, options: options) file = downloader.perform uploader = UploadService.new(project, file, FileUploader).execute uploader.to_h[:url] + rescue ::Gitlab::GithubImport::AttachmentsDownloader::UnsupportedAttachmentError + attachment.url ensure downloader&.delete end + def options + { + headers: { + 'Authorization' => "Bearer #{client.octokit.access_token}" + } + } + end + def update_note_record(text) case note_text.record_type when ::Release.name diff --git a/lib/gitlab/github_import/markdown/attachment.rb b/lib/gitlab/github_import/markdown/attachment.rb index e270cfba619..0d8f3196719 100644 --- a/lib/gitlab/github_import/markdown/attachment.rb +++ b/lib/gitlab/github_import/markdown/attachment.rb @@ -57,7 +57,8 @@ module Gitlab def github_url?(url, docs: false, media: false) if media - url.start_with?(::Gitlab::GithubImport::MarkdownText::GITHUB_MEDIA_CDN) + url.start_with?(::Gitlab::GithubImport::MarkdownText.github_url, + ::Gitlab::GithubImport::MarkdownText::GITHUB_MEDIA_CDN) elsif docs url.start_with?(::Gitlab::GithubImport::MarkdownText.github_url) end @@ -65,6 +66,9 @@ module Gitlab def whitelisted_type?(url, docs: false, media: false) if media + # We do not know the file extension type from the /assets markdown + return true if url.start_with?(::Gitlab::GithubImport::MarkdownText.github_url) + MEDIA_TYPES.any? { |type| url.end_with?(type) } elsif docs DOC_TYPES.any? { |type| url.end_with?(type) } @@ -91,8 +95,11 @@ module Gitlab ) end - def media? - url.start_with?(::Gitlab::GithubImport::MarkdownText::GITHUB_MEDIA_CDN) + def media?(import_source) + url.start_with?( + "#{::Gitlab::GithubImport::MarkdownText.github_url}/#{import_source}/assets", + ::Gitlab::GithubImport::MarkdownText::GITHUB_MEDIA_CDN + ) end def inspect diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb index 7ee64b2abac..88e91800cee 100644 --- a/lib/gitlab/github_import/object_counter.rb +++ b/lib/gitlab/github_import/object_counter.rb @@ -14,6 +14,8 @@ module Gitlab CACHING = Gitlab::Cache::Import::Caching + IMPORT_CACHING_TIMEOUT = 2.weeks.to_i + class << self # Increments the project and the global counters if the given value is >= 1 def increment(project, object_type, operation, value: 1) @@ -36,7 +38,7 @@ module Gitlab # After import is completed we store this information in project's import_checksums return cached_summary if cached_summary != EMPTY_SUMMARY || project.import_state.blank? - project.import_state.in_progress? ? cached_summary : project.import_checksums + project.import_state.completed? ? project.import_checksums : cached_summary end private @@ -50,7 +52,7 @@ module Gitlab .sort .each do |counter| object_type = counter.split('/').last - result[operation][object_type] = CACHING.read_integer(counter) + result[operation][object_type] = CACHING.read_integer(counter) || 0 end end end @@ -84,11 +86,11 @@ module Gitlab add_counter_to_list(project, operation, counter_key) - CACHING.increment_by(counter_key, value) + CACHING.increment_by(counter_key, value, timeout: IMPORT_CACHING_TIMEOUT) end def add_counter_to_list(project, operation, key) - CACHING.set_add(counter_list_key(project, operation), key) + CACHING.set_add(counter_list_key(project, operation), key, timeout: IMPORT_CACHING_TIMEOUT) end def counter_list_key(project, operation) diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 57365ebe206..1832f071a44 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -28,6 +28,17 @@ module Gitlab EMAIL_FOR_USERNAME_CACHE_KEY = 'github-import/user-finder/email-for-username/%s' + # The base cache key to use for caching the user ETAG response headers + USERNAME_ETAG_CACHE_KEY = 'github-import/user-finder/user-etag/%s' + + # The base cache key to store whether an email has been fetched for a project + EMAIL_FETCHED_FOR_PROJECT_CACHE_KEY = 'github-import/user-finder/%{project}/email-fetched/%{username}' + + EMAIL_API_CALL_LOGGING_MESSAGE = { + true => 'Fetching email from GitHub with ETAG header', + false => 'Fetching email from GitHub' + }.freeze + # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` def initialize(project, client) @@ -109,24 +120,39 @@ module Gitlab id_for_github_id(id) || id_for_github_email(email) end - # Find the public email of a given username in GitHub. The public email is cached to avoid multiple calls to - # GitHub. In case the username does not exist or the public email is nil, a blank value is cached to also prevent - # multiple calls to GitHub. + # Find the public email of a given username in GitHub. + # The email is cached to avoid multiple calls to GitHub. The cache is shared among all projects. + # If the email was not found, a blank email is cached. + # If the email is blank, we attempt to fetch it from GitHub using an ETAG request once for every project. + + # @param username [String] The username of the GitHub user. # # @return [String] If public email is found # @return [Nil] If public email or username does not exist def email_for_github_username(username) - cache_key = EMAIL_FOR_USERNAME_CACHE_KEY % username - email = Gitlab::Cache::Import::Caching.read(cache_key) + email = read_email_from_cache(username) + + if email.blank? && !email_fetched_for_project?(username) + # If an ETAG is available, make an API call with the ETAG. + # Only make a rate-limited API call if the ETAG is not available and the email is nil. + etag = read_etag_from_cache(username) + email = fetch_email_from_github(username, etag: etag) || email - if email.nil? - user = client.user(username) - email = Gitlab::Cache::Import::Caching.write(cache_key, user[:email].to_s, timeout: timeout(user[:email])) + cache_email!(username, email) + cache_etag!(username) if email.blank? && etag.nil? + + # If a non-blank email is cached, we don't need the ETAG or project check caches. + # Otherwise, indicate that the project has been checked. + if email.present? + clear_caches!(username) + else + set_project_as_checked!(username) + end end email.presence rescue ::Octokit::NotFound - Gitlab::Cache::Import::Caching.write(cache_key, '') + cache_email!(username, '') nil end @@ -192,12 +218,66 @@ module Gitlab private - def timeout(email) - if email - Gitlab::Cache::Import::Caching::TIMEOUT - else - Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT - end + def read_email_from_cache(username) + Gitlab::Cache::Import::Caching.read(email_cache_key(username)) + end + + def read_etag_from_cache(username) + Gitlab::Cache::Import::Caching.read(etag_cache_key(username)) + end + + def email_fetched_for_project?(username) + email_fetched_for_project_cache_key = email_fetched_for_project_cache_key(username) + Gitlab::Cache::Import::Caching.read(email_fetched_for_project_cache_key) + end + + def fetch_email_from_github(username, etag: nil) + log(EMAIL_API_CALL_LOGGING_MESSAGE[etag.present?], username: username) + user = client.user(username, { headers: { 'If-None-Match' => etag }.compact }) + + user[:email] || '' if user + end + + def cache_email!(username, email) + return unless email + + Gitlab::Cache::Import::Caching.write(email_cache_key(username), email) + end + + def cache_etag!(username) + etag = client.octokit.last_response.headers[:etag] + Gitlab::Cache::Import::Caching.write(etag_cache_key(username), etag) + end + + def set_project_as_checked!(username) + Gitlab::Cache::Import::Caching.write(email_fetched_for_project_cache_key(username), 1) + end + + def clear_caches!(username) + Gitlab::Cache::Import::Caching.expire(etag_cache_key(username), 0) + Gitlab::Cache::Import::Caching.expire(email_fetched_for_project_cache_key(username), 0) + end + + def email_cache_key(username) + EMAIL_FOR_USERNAME_CACHE_KEY % username + end + + def etag_cache_key(username) + USERNAME_ETAG_CACHE_KEY % username + end + + def email_fetched_for_project_cache_key(username) + format(EMAIL_FETCHED_FOR_PROJECT_CACHE_KEY, project: project.id, username: username) + end + + def log(message, username: nil) + Gitlab::Import::Logger.info( + import_type: :github, + project_id: project.id, + class: self.class.name, + username: username, + message: message + ) end end end diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index 9161a1a138f..5745b65b6fc 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -34,7 +34,8 @@ module Gitlab DESIGN = ::Gitlab::GlRepository::RepoType.new( name: :design, access_checker_class: ::Gitlab::GitAccessDesign, - repository_resolver: -> (project) { project.find_or_create_design_management_repository.repository }, + repository_resolver: -> (design_management_repository) { design_management_repository.repository }, + project_resolver: -> (design_management_repository) { design_management_repository&.project }, suffix: :design, container_class: DesignManagement::Repository ).freeze diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 26b0ff86f67..0c0d9bafa85 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -2,6 +2,18 @@ module Gitlab class GlRepository + class ContainerClassMismatchError < StandardError + def initialize(container_class, repo_type) + @container_class = container_class + @repo_type = repo_type + end + + def message + "Expected container class to be #{@repo_type.container_class} for " \ + "repo type #{@repo_type.name}, but found #{@container_class} instead." + end + end + class RepoType attr_reader :name, :access_checker_class, @@ -53,13 +65,14 @@ module Gitlab end def repository_for(container) + check_container(container) return unless container - repository_resolver.call(select_container(container)) + repository_resolver.call(container) end def project_for(container) - return select_container(container) unless project_resolver + return container unless project_resolver project_resolver.call(container) end @@ -74,13 +87,20 @@ module Gitlab private - def select_container(container) - container.is_a?(::DesignManagement::Repository) ? container.project : container - end - def default_container_class Project end + + def check_container(container) + # Don't check container for wiki or project because these repo types + # accept several container types. + return if wiki? || project? + + return unless container.present? && container_class.present? + return if container.is_a?(container_class) + + raise ContainerClassMismatchError.new(container.class.name, self) + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 27c4ec2f7be..eefa23142af 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -17,6 +17,8 @@ module Gitlab gon.markdown_surround_selection = current_user&.markdown_surround_selection gon.markdown_automatic_lists = current_user&.markdown_automatic_lists + add_browsersdk_tracking + if Gitlab.config.sentry.enabled gon.sentry_dsn = Gitlab.config.sentry.clientside_dsn gon.sentry_environment = Gitlab.config.sentry.environment @@ -28,6 +30,7 @@ module Gitlab current_user) && Gitlab::CurrentSettings.sentry_enabled gon.sentry_dsn = Gitlab::CurrentSettings.sentry_clientside_dsn gon.sentry_environment = Gitlab::CurrentSettings.sentry_environment + gon.sentry_clientside_traces_sample_rate = Gitlab::CurrentSettings.sentry_clientside_traces_sample_rate end gon.recaptcha_api_server_url = ::Recaptcha.configuration.api_server_url @@ -52,6 +55,7 @@ module Gitlab gon.uf_error_prefix = ::Gitlab::Utils::ErrorMessage::UF_ERROR_PREFIX gon.pat_prefix = Gitlab::CurrentSettings.current_application_settings.personal_access_token_prefix gon.use_new_navigation = NavHelper.show_super_sidebar?(current_user) + gon.keyboard_shortcuts_enabled = current_user ? current_user.keyboard_shortcuts_enabled : true gon.diagramsnet_url = Gitlab::CurrentSettings.diagramsnet_url if Gitlab::CurrentSettings.diagramsnet_enabled @@ -71,12 +75,11 @@ module Gitlab push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) push_frontend_feature_flag(:unbatch_graphql_queries, current_user) - push_frontend_feature_flag(:command_palette, current_user) + push_frontend_feature_flag(:server_side_frecent_namespaces, current_user) # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248 push_frontend_feature_flag(:remove_monitor_metrics) push_frontend_feature_flag(:gitlab_duo, current_user) push_frontend_feature_flag(:custom_emoji) - push_frontend_feature_flag(:super_sidebar_flyout_menus, current_user) end # Exposes the state of a feature flag to the frontend code. @@ -116,6 +119,14 @@ module Gitlab URI.join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s end + + def add_browsersdk_tracking + return unless Gitlab.com? && Feature.enabled?(:browsersdk_tracking) + return if ENV['GITLAB_ANALYTICS_URL'].blank? || ENV['GITLAB_ANALYTICS_ID'].blank? + + gon.analytics_url = ENV['GITLAB_ANALYTICS_URL'] + gon.analytics_id = ENV['GITLAB_ANALYTICS_ID'] + end end end diff --git a/lib/gitlab/graphql/deprecations.rb b/lib/gitlab/graphql/deprecations.rb index 9cd8462f2e8..221b19bf8a3 100644 --- a/lib/gitlab/graphql/deprecations.rb +++ b/lib/gitlab/graphql/deprecations.rb @@ -27,7 +27,6 @@ module Gitlab end # GitLab allows items to be marked as "alpha", which leverages GraphQL deprecations. - # TODO remove deprecation_args = kwargs.extract!(:alpha, :deprecated) self.deprecation = Deprecation.parse(**deprecation_args) diff --git a/lib/gitlab/graphql/deprecations/deprecation.rb b/lib/gitlab/graphql/deprecations/deprecation.rb index dfcca5ee75b..0cf555b0e34 100644 --- a/lib/gitlab/graphql/deprecations/deprecation.rb +++ b/lib/gitlab/graphql/deprecations/deprecation.rb @@ -5,7 +5,7 @@ module Gitlab module Deprecations class Deprecation REASON_RENAMED = :renamed - REASON_ALPHA = :alpha # TODO remove support in this class + REASON_ALPHA = :alpha REASONS = { REASON_RENAMED => 'This was renamed.', @@ -27,7 +27,7 @@ module Gitlab return unless options if alpha - raise ArgumentError, '`experiment` and `deprecated` arguments cannot be passed at the same time' \ + raise ArgumentError, '`alpha` and `deprecated` arguments cannot be passed at the same time' \ if deprecated options[:reason] = :alpha diff --git a/lib/gitlab/graphql/query_analyzers/ast/recursion_analyzer.rb b/lib/gitlab/graphql/query_analyzers/ast/recursion_analyzer.rb index 4e90e4c912f..34856cfc238 100644 --- a/lib/gitlab/graphql/query_analyzers/ast/recursion_analyzer.rb +++ b/lib/gitlab/graphql/query_analyzers/ast/recursion_analyzer.rb @@ -7,7 +7,7 @@ module Gitlab module QueryAnalyzers module AST class RecursionAnalyzer < GraphQL::Analysis::AST::Analyzer - IGNORED_FIELDS = %w(node edges nodes ofType).freeze + IGNORED_FIELDS = %w[node edges nodes ofType].freeze RECURSION_THRESHOLD = 2 def initialize(query) diff --git a/lib/gitlab/hook_data/user_builder.rb b/lib/gitlab/hook_data/user_builder.rb index 54f03b863e5..d450bf80cd1 100644 --- a/lib/gitlab/hook_data/user_builder.rb +++ b/lib/gitlab/hook_data/user_builder.rb @@ -49,5 +49,3 @@ module Gitlab end end end - -Gitlab::HookData::UserBuilder.prepend_mod_with('Gitlab::HookData::UserBuilder') diff --git a/lib/gitlab/hotlinking_detector.rb b/lib/gitlab/hotlinking_detector.rb index e81983cd014..bfcc1735cb0 100644 --- a/lib/gitlab/hotlinking_detector.rb +++ b/lib/gitlab/hotlinking_detector.rb @@ -2,11 +2,11 @@ module Gitlab class HotlinkingDetector - IMAGE_FORMATS = %w(image/jpeg image/apng image/png image/webp image/svg+xml image/*).freeze - MEDIA_FORMATS = %w(video/webm video/ogg video/* application/ogg audio/webm audio/ogg audio/wav audio/*).freeze - CSS_FORMATS = %w(text/css).freeze + IMAGE_FORMATS = %w[image/jpeg image/apng image/png image/webp image/svg+xml image/*].freeze + MEDIA_FORMATS = %w[video/webm video/ogg video/* application/ogg audio/webm audio/ogg audio/wav audio/*].freeze + CSS_FORMATS = %w[text/css].freeze INVALID_FORMATS = (IMAGE_FORMATS + MEDIA_FORMATS + CSS_FORMATS).freeze - INVALID_FETCH_MODES = %w(cors no-cors websocket).freeze + INVALID_FETCH_MODES = %w[cors no-cors websocket].freeze class << self def intercept_hotlinking?(request) diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 8b19611e5c0..feb54fcca0c 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -1,9 +1,13 @@ # frozen_string_literal: true -# This class is used as a proxy for all outbounding http connection -# coming from callbacks, services and hooks. The direct use of the HTTParty -# is discouraged because it can lead to several security problems, like SSRF -# calling internal IP or services. +# +# IMPORTANT: With the new development of the 'gitlab-http' gem (https://gitlab.com/gitlab-org/gitlab/-/issues/415686), +# no additional change should be implemented in this class. This class will be removed after migrating all +# the usages to the new gem. +# + +require_relative 'http_connection_adapter' + module Gitlab class HTTP BlockedUrlError = Class.new(StandardError) @@ -42,7 +46,7 @@ module Gitlab alias_method :httparty_perform_request, :perform_request end - connection_adapter HTTPConnectionAdapter + connection_adapter ::Gitlab::HTTPConnectionAdapter def self.perform_request(http_method, path, options, &block) raise_if_blocked_by_silent_mode(http_method) diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index afb740a902b..822b8a9f8d9 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -18,6 +18,8 @@ # to read header data. It is a modified version of Net::BufferedIO that # raises a timeout error if reading header data takes too much time. +require_relative 'utils/override' + module Gitlab class HTTPConnectionAdapter < HTTParty::ConnectionAdapter extend ::Gitlab::Utils::Override diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 6b154c7033f..f8e7e66a8a5 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -48,24 +48,24 @@ module Gitlab 'de' => 96, 'en' => 100, 'eo' => 0, - 'es' => 29, + 'es' => 28, 'fil_PH' => 0, 'fr' => 99, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, 'ja' => 99, - 'ko' => 20, + 'ko' => 23, 'nb_NO' => 21, 'nl_NL' => 0, 'pl_PL' => 3, 'pt_BR' => 55, - 'ro_RO' => 78, + 'ro_RO' => 76, 'ru' => 22, - 'si_LK' => 9, - 'tr_TR' => 9, - 'uk' => 52, - 'zh_CN' => 98, + 'si_LK' => 12, + 'tr_TR' => 8, + 'uk' => 51, + 'zh_CN' => 99, 'zh_HK' => 1, 'zh_TW' => 99 }.freeze diff --git a/lib/gitlab/import.rb b/lib/gitlab/import.rb new file mode 100644 index 00000000000..3f3692d4c0c --- /dev/null +++ b/lib/gitlab/import.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Gitlab + module Import + JOB_WAITER_TTL = 1.week.to_i + end +end diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index 986191bdb6b..62bd650c1d5 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -15,7 +15,6 @@ module Gitlab include Gitlab::Utils::StrongMemoize BATCH_SIZE = 100 - MIN_RECORDS_SIZE = 1 attr_reader :invalid_subrelations @@ -82,7 +81,7 @@ module Gitlab subrelation = relation_object.public_send(definition) association = relation_object.class.reflect_on_association(definition) - next unless association&.collection? && subrelation.size > MIN_RECORDS_SIZE + next unless association&.collection? collection_subrelations[definition] = subrelation.records diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 924ca4e83ea..dfe0815f0a0 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -37,7 +37,7 @@ module Gitlab def gzip_with_options(dir:, filename:, options: nil) filepath = File.join(dir, filename) - cmd = %W(gzip #{filepath}) + cmd = %W[gzip #{filepath}] cmd << "-#{options}" if options _, status = Gitlab::Popen.popen(cmd) @@ -68,6 +68,8 @@ module Gitlab File.open(upload_path, 'wb') do |file| current_size = 0 + # When migrating from Gitlab::HTTP to Gitlab:HTTP_V2, we need to pass `extra_allowed_uris` as an option + # instead of `allow_object_storage`. Gitlab::HTTP.get(url, stream_body: true, allow_object_storage: true) do |fragment| if [301, 302, 303, 307].include?(fragment.code) Gitlab::Import::Logger.warn(message: "received redirect fragment", fragment_code: fragment.code) @@ -87,12 +89,12 @@ module Gitlab end def tar_with_options(archive:, dir:, options:) - execute_cmd(%W(tar -#{options} #{archive} -C #{dir} .)) + execute_cmd(%W[tar -#{options} #{archive} -C #{dir} .]) end def untar_with_options(archive:, dir:, options:) - execute_cmd(%W(tar -#{options} #{archive} -C #{dir})) - execute_cmd(%W(chmod -R #{UNTAR_MASK} #{dir})) + execute_cmd(%W[tar -#{options} #{archive} -C #{dir}]) + execute_cmd(%W[chmod -R #{UNTAR_MASK} #{dir}]) clean_extraction_dir!(dir) end diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 3609df89958..13510cb43ca 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -5,13 +5,10 @@ module Gitlab class DecompressedArchiveSizeValidator include Gitlab::Utils::StrongMemoize - TIMEOUT_LIMIT = 210.seconds - ServiceError = Class.new(StandardError) - def initialize(archive_path:, max_bytes: self.class.max_bytes) + def initialize(archive_path:) @archive_path = archive_path - @max_bytes = max_bytes end def valid? @@ -20,10 +17,6 @@ module Gitlab end end - def self.max_bytes - Gitlab::CurrentSettings.current_application_settings.max_decompressed_archive_size.megabytes - end - private def validate @@ -32,7 +25,7 @@ module Gitlab validate_archive_path - Timeout.timeout(TIMEOUT_LIMIT) do + Timeout.timeout(timeout) do stderr_r, stderr_w = IO.pipe stdout, wait_threads = Open3.pipeline_r(*command, pgroup: true, err: stderr_w) @@ -51,7 +44,7 @@ module Gitlab if status.success? result = stdout.readline - if @max_bytes > 0 && result.to_i > @max_bytes + if max_bytes > 0 && result.to_i > max_bytes valid_archive = false log_error('Decompressed archive size limit reached') @@ -70,7 +63,7 @@ module Gitlab valid_archive rescue Timeout::Error - log_error('Timeout reached during archive decompression') + log_error("Timeout of #{timeout} seconds reached during archive decompression") pgrps.each { |pgrp| Process.kill(-1, pgrp) } if pgrps @@ -107,6 +100,14 @@ module Gitlab import_upload_archive_size: archive_size ) end + + def timeout + Gitlab::CurrentSettings.current_application_settings.decompress_archive_file_timeout + end + + def max_bytes + Gitlab::CurrentSettings.current_application_settings.max_decompressed_archive_size.megabytes + end end end end diff --git a/lib/gitlab/import_export/json/ndjson_writer.rb b/lib/gitlab/import_export/json/ndjson_writer.rb index e303ac6eefa..60ae163cad5 100644 --- a/lib/gitlab/import_export/json/ndjson_writer.rb +++ b/lib/gitlab/import_export/json/ndjson_writer.rb @@ -44,12 +44,11 @@ module Gitlab def with_file(*path) file_path = File.join(@dir_path, *path) - raise ArgumentError, "The #{file_path} already exist" if File.exist?(file_path) # ensure that path is created mkdir_p(File.dirname(file_path)) - File.open(file_path, "wb") do |file| + File.open(file_path, "ab+") do |file| yield(file) end end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 2c64ca53f76..b0951f24628 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -75,10 +75,12 @@ module Gitlab def serialize_many_relations(key, records, options) log_relation_export(key, records.size) - enumerator = Enumerator.new do |items| - key_preloads = preloads&.dig(key) + key_preloads = preloads&.dig(key) + + batch(records, key) do |batch| + next if batch.empty? - batch(records, key) do |batch| + batch_enumerator = Enumerator.new do |items| batch = batch.preload(key_preloads) if key_preloads batch.each do |record| @@ -91,9 +93,11 @@ module Gitlab after_read_callback(record) end end - end - json_writer.write_relation_array(@exportable_path, key, enumerator) + json_writer.write_relation_array(@exportable_path, key, batch_enumerator) + + Gitlab::SafeRequestStore.clear! + end end def exportable_json_record(record, options, key) diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 850c89c1fb1..1b04e55e0c7 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -116,7 +116,6 @@ tree: - :project_badges - :ci_cd_settings - :error_tracking_setting - - :metrics_setting - boards: - lists: - label: @@ -156,10 +155,6 @@ included_attributes: - :group_runners_enabled - :runner_token_expiration_interval - :default_git_depth - metrics_setting: - - :dashboard_timezone - - :external_dashboard_url - - :project_id project_badges: - :created_at - :image_url diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 7af65235492..840621a94a2 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -38,7 +38,6 @@ module Gitlab ci_cd_settings: 'ProjectCiCdSetting', error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting', links: 'Releases::Link', - metrics_setting: 'ProjectMetricsSetting', commit_author: 'MergeRequest::DiffCommitUser', committer: 'MergeRequest::DiffCommitUser', merge_request_diff_commits: 'MergeRequestDiffCommit', @@ -180,7 +179,7 @@ module Gitlab # When author is not present for source release set the author as ghost user. if @relation_hash['author_id'].blank? - @relation_hash['author_id'] = User.select(:id).ghost.id + @relation_hash['author_id'] = Users::Internal.ghost.id end end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index d7d262501de..1861a92100e 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -42,7 +42,7 @@ module Gitlab def ensure_repository_does_not_exist! if repository.exists? shared.logger.info( - message: %{Deleting existing "#{repository.disk_path}" to re-import it.} + message: %(Deleting existing "#{repository.disk_path}" to re-import it.) ) Repositories::DestroyService.new(repository).execute diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 37bcc53019f..fec8b3a7708 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -20,9 +20,6 @@ module Gitlab ImportSource.new('manifest', 'Manifest file', nil) ].freeze - LEGACY_IMPORT_TABLE = IMPORT_TABLE.deep_dup - LEGACY_IMPORT_TABLE[2].importer = Gitlab::BitbucketServerImport::Importer - class << self prepend_mod_with('Gitlab::ImportSources') # rubocop: disable Cop/InjectEnterpriseEditionModule @@ -47,9 +44,15 @@ module Gitlab end def import_table - return IMPORT_TABLE if Feature.enabled?(:bitbucket_server_parallel_importer) + bitbucket_parallel_enabled = Feature.enabled?(:bitbucket_parallel_importer) + + return IMPORT_TABLE unless bitbucket_parallel_enabled + + import_table = IMPORT_TABLE.deep_dup + + import_table[1].importer = Gitlab::BitbucketImport::ParallelImporter if bitbucket_parallel_enabled - LEGACY_IMPORT_TABLE + import_table end end end diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 70aaa59f912..e39bbb36680 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -110,6 +110,16 @@ module Gitlab @request_counter.increment({ storage: storage_key }, amount) end + def instance_count_pipelined_request(size) + @pipeline_size_histogram ||= Gitlab::Metrics.histogram( + :gitlab_redis_client_requests_pipelined_commands, + 'Client side Redis request pipeline size, per Redis server', + {}, + [10, 100, 1000, 10_000] + ) + @pipeline_size_histogram.observe({ storage: storage_key }, size) + end + def instance_count_exception(ex) # This metric is meant to give a client side view of how the Redis # server is doing. Redis itself does not expose error counts. This diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index b3fbe30e583..20ba1ab82a7 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -12,7 +12,7 @@ module Gitlab end def call_pipeline(pipeline) - instrument_call(pipeline.commands) do + instrument_call(pipeline.commands, true) do super end end @@ -30,9 +30,10 @@ module Gitlab private - def instrument_call(commands) + def instrument_call(commands, pipelined = false) start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined instrumentation_class.instance_count_request(commands.size) + instrumentation_class.instance_count_pipelined_request(commands.size) if pipelined if !instrumentation_class.redis_cluster_validate!(commands) && ::RequestStore.active? instrumentation_class.increment_cross_slot_request_count diff --git a/lib/gitlab/jira_import/metadata_collector.rb b/lib/gitlab/jira_import/metadata_collector.rb index 090b95ac14a..de5d636cc5b 100644 --- a/lib/gitlab/jira_import/metadata_collector.rb +++ b/lib/gitlab/jira_import/metadata_collector.rb @@ -11,8 +11,8 @@ module Gitlab end def execute - add_field(%w(issuetype name), 'Issue type') - add_field(%w(priority name), 'Priority') + add_field(%w[issuetype name], 'Issue type') + add_field(%w[priority name], 'Priority') add_field('environment', 'Environment') add_field('duedate', 'Due date') add_parent diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index 880b112d815..e53bfb40654 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -22,14 +22,16 @@ module Gitlab STARTED_METRIC = :gitlab_job_waiter_started_total TIMEOUTS_METRIC = :gitlab_job_waiter_timeouts_total - def self.notify(key, jid) + # This TTL needs to be long enough to allow whichever Sidekiq job calls + # JobWaiter#wait to reach BLPOP. + DEFAULT_TTL = 6.hours.to_i + + def self.notify(key, jid, ttl: DEFAULT_TTL) Gitlab::Redis::SharedState.with do |redis| # Use a Redis MULTI transaction to ensure we always set an expiry redis.multi do |multi| multi.lpush(key, jid) - # This TTL needs to be long enough to allow whichever Sidekiq job calls - # JobWaiter#wait to reach BLPOP. - multi.expire(key, 6.hours.to_i) + multi.expire(key, ttl) end end end @@ -42,6 +44,10 @@ module Gitlab "#{KEY_PREFIX}:#{SecureRandom.uuid}" end + def self.delete_key(key) + Gitlab::Redis::SharedState.with { |redis| redis.del(key) } if key?(key) + end + attr_reader :key, :finished, :worker_label attr_accessor :jobs_remaining diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 255d8802c1c..de391c45121 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -51,7 +51,7 @@ module Gitlab # Legacy code path. Will be removed when all distributions provide a sane default here uri = URI.join(external_url, K8S_PROXY_PATH) - uri.scheme = uri.scheme.in?(%w(grpcs wss)) ? 'https' : 'http' + uri.scheme = uri.scheme.in?(%w[grpcs wss]) ? 'https' : 'http' uri.to_s end diff --git a/lib/gitlab/kroki.rb b/lib/gitlab/kroki.rb index 5fa77c1f1ba..84dc081d9d7 100644 --- a/lib/gitlab/kroki.rb +++ b/lib/gitlab/kroki.rb @@ -14,8 +14,8 @@ module Gitlab packetdiag rackdiag ].freeze - DIAGRAMS_FORMATS = (::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES - %w(mermaid)).freeze - DIAGRAMS_FORMATS_WO_PLANTUML = (DIAGRAMS_FORMATS - %w(plantuml)).freeze + DIAGRAMS_FORMATS = (::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES - %w[mermaid]).freeze + DIAGRAMS_FORMATS_WO_PLANTUML = (DIAGRAMS_FORMATS - %w[plantuml]).freeze # Get the list of diagram formats that are currently enabled # diff --git a/lib/gitlab/kubernetes/kubectl_cmd.rb b/lib/gitlab/kubernetes/kubectl_cmd.rb index f3ac19e210a..28a010ec191 100644 --- a/lib/gitlab/kubernetes/kubectl_cmd.rb +++ b/lib/gitlab/kubernetes/kubectl_cmd.rb @@ -5,23 +5,23 @@ module Gitlab module KubectlCmd class << self def delete(*args) - %w(kubectl delete).concat(args).shelljoin + %w[kubectl delete].concat(args).shelljoin end def apply_file(filename, *args) raise ArgumentError, "filename is not present" unless filename.present? - %w(kubectl apply -f).concat([filename], args).shelljoin + %w[kubectl apply -f].concat([filename], args).shelljoin end def delete_crds_from_group(group) - api_resources_args = %w(-o name --api-group).push(group) + api_resources_args = %w[-o name --api-group].push(group) PodCmd.retry_command(api_resources(*api_resources_args) + " | xargs -r " + delete('--ignore-not-found', 'crd')) end def api_resources(*args) - %w(kubectl api-resources).concat(args).shelljoin + %w[kubectl api-resources].concat(args).shelljoin end end end diff --git a/lib/gitlab/logger.rb b/lib/gitlab/logger.rb index 53ad0d9cb4d..cc172bb9b85 100644 --- a/lib/gitlab/logger.rb +++ b/lib/gitlab/logger.rb @@ -27,7 +27,7 @@ module Gitlab return [] unless File.readable?(path) - tail_output, _ = Gitlab::Popen.popen(%W(tail -n 2000 #{path})) + tail_output, _ = Gitlab::Popen.popen(%W[tail -n 2000 #{path}]) tail_output.split("\n") end diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index f8ec58cf217..6f180f89db0 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -6,7 +6,7 @@ module Gitlab include ::Gitlab::Logging::CloudflareHelper LIMITED_ARRAY_SENTINEL = { key: 'truncated', value: '...' }.freeze - IGNORE_PARAMS = Set.new(%w(controller action format)).freeze + IGNORE_PARAMS = Set.new(%w[controller action format]).freeze KNOWN_PAYLOAD_PARAMS = [:remote_ip, :user_id, :username, :ua, :queue_duration_s, :response_bytes, :etag_route, :request_urgency, :target_duration_s] + CLOUDFLARE_CUSTOM_HEADERS.values diff --git a/lib/gitlab/manifest_import/metadata.rb b/lib/gitlab/manifest_import/metadata.rb index 81711be729e..b2871b96ad6 100644 --- a/lib/gitlab/manifest_import/metadata.rb +++ b/lib/gitlab/manifest_import/metadata.rb @@ -40,13 +40,9 @@ module Gitlab "#{KEY_PREFIX}:{#{user.id}}:#{field}" end - def key_for(field) - "#{KEY_PREFIX}:#{user.id}:#{field}" - end - def redis_get(field) Gitlab::Redis::SharedState.with do |redis| - redis.get(hashtag_key_for(field)) || redis.get(key_for(field)) + redis.get(hashtag_key_for(field)) end end end diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb index 54095461dd4..7ffec7b0141 100644 --- a/lib/gitlab/metrics/background_transaction.rb +++ b/lib/gitlab/metrics/background_transaction.rb @@ -8,7 +8,7 @@ module Gitlab # metadata such as endpoint_id, queue, and feature category. class BackgroundTransaction < Transaction THREAD_KEY = :_gitlab_metrics_background_transaction - BASE_LABEL_KEYS = %i(queue endpoint_id feature_category).freeze + BASE_LABEL_KEYS = %i[queue endpoint_id feature_category].freeze class << self def current diff --git a/lib/gitlab/metrics/dashboard/cache.rb b/lib/gitlab/metrics/dashboard/cache.rb deleted file mode 100644 index 54b5250d209..00000000000 --- a/lib/gitlab/metrics/dashboard/cache.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -require 'set' - -module Gitlab - module Metrics - module Dashboard - class Cache - CACHE_KEYS = 'all_cached_metric_dashboards' - - class << self - # This class method (Gitlab::Metrics::Dashboard::Cache.fetch) can be used - # when the key does not need to be deleted by `delete_all!`. - # For example, out of the box dashboard caches do not need to be deleted. - delegate :fetch, to: :"Rails.cache" - - alias_method :for, :new - end - - def initialize(project) - @project = project - end - - # Stores a dashboard in the cache, documenting the key - # so the cache can be cleared in bulk at another time. - def fetch(key) - register_key(key) - - Rails.cache.fetch(key) { yield } - end - - # Resets all dashboard caches, such that all - # dashboard content will be loaded from source on - # subsequent dashboard calls. - def delete_all! - all_keys.each { |key| Rails.cache.delete(key) } - - Rails.cache.delete(catalog_key) - end - - private - - def register_key(key) - new_keys = all_keys.add(key).to_a.join('|') - - Rails.cache.write(catalog_key, new_keys) - end - - def all_keys - keys = Rails.cache.read(catalog_key)&.split('|') - Set.new(keys) - end - - # One key to store them all... - # This key is used to store the names of all the keys that contain this - # project's dashboards. - def catalog_key - "#{CACHE_KEYS}_#{@project.id}" - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/errors.rb b/lib/gitlab/metrics/dashboard/errors.rb deleted file mode 100644 index 1a951172f74..00000000000 --- a/lib/gitlab/metrics/dashboard/errors.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -# Central point for managing errors from within the metrics -# dashboard module. Handles errors from dashboard retrieval -# and processing steps, as well as defines shared error classes. -module Gitlab - module Metrics - module Dashboard - module Errors - DashboardProcessingError = Class.new(StandardError) - PanelNotFoundError = Class.new(StandardError) - MissingIntegrationError = Class.new(StandardError) - LayoutError = Class.new(DashboardProcessingError) - MissingQueryError = Class.new(DashboardProcessingError) - - NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError - - def handle_errors(error) - case error - when DashboardProcessingError - error(error.message, :unprocessable_entity) - when NOT_FOUND_ERROR - error(_("%{dashboard_path} could not be found.") % { dashboard_path: dashboard_path }, :not_found) - when PanelNotFoundError - error(error.message, :not_found) - when ::Grafana::Client::Error - error(error.message, :service_unavailable) - when MissingIntegrationError - error(_('Proxy support for this API is not available currently'), :bad_request) - else - raise error - end - end - - def panels_not_found!(opts) - raise PanelNotFoundError, _("No panels matching properties %{opts}") % { opts: opts } - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/processor.rb b/lib/gitlab/metrics/dashboard/processor.rb deleted file mode 100644 index 9566e5afb9a..00000000000 --- a/lib/gitlab/metrics/dashboard/processor.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - # Responsible for processesing a dashboard hash, inserting - # relevant DB records & sorting for proper rendering in - # the UI. These includes shared metric info, custom metrics - # info, and alerts (only in EE). - class Processor - def initialize(project, dashboard, sequence, params) - @project = project - @dashboard = dashboard - @sequence = sequence - @params = params - end - - # Returns a new dashboard hash with the results of - # running transforms on the dashboard. - # @return [Hash, nil] - def process - return unless @dashboard - - @dashboard.deep_symbolize_keys.tap do |dashboard| - @sequence.each do |stage| - stage.new(@project, dashboard, @params).transform! - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb b/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb deleted file mode 100644 index 8b791e110ba..00000000000 --- a/lib/gitlab/metrics/dashboard/repo_dashboard_finder.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -# Provides methods to list and read dashboard yaml files from a project's repository. -module Gitlab - module Metrics - module Dashboard - class RepoDashboardFinder - DASHBOARD_ROOT = ".gitlab/dashboards" - DASHBOARD_EXTENSION = '.yml' - - class << self - # Returns list of all user-defined dashboard paths. Used to populate - # Repository model cache (Repository#user_defined_metrics_dashboard_paths). - # Also deletes all dashboard cache entries. - # @return [Array] ex) ['.gitlab/dashboards/dashboard1.yml'] - def list_dashboards(project) - Gitlab::Metrics::Dashboard::Cache.for(project).delete_all! - - file_finder(project).list_files_for(DASHBOARD_ROOT) - end - - # Reads the given dashboard from repository, and returns the content as a string. - # @return [String] - def read_dashboard(project, dashboard_path) - file_finder(project).read(dashboard_path) - end - - private - - def file_finder(project) - Gitlab::Template::Finders::RepoTemplateFinder.new(project, DASHBOARD_ROOT, DASHBOARD_EXTENSION) - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/base_stage.rb b/lib/gitlab/metrics/dashboard/stages/base_stage.rb deleted file mode 100644 index b869a633030..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/base_stage.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class BaseStage - attr_reader :project, :dashboard, :params - - def initialize(project, dashboard, params) - @project = project - @dashboard = dashboard - @params = params - end - - # Entry-point to the stage - def transform! - raise NotImplementedError - end - - protected - - def missing_panel_groups! - raise Errors::LayoutError, 'Top-level key :panel_groups must be an array' - end - - def missing_panels! - raise Errors::LayoutError, 'Each "panel_group" must define an array :panels' - end - - def missing_metrics! - raise Errors::LayoutError, 'Each "panel" must define an array :metrics' - end - - def for_metrics - missing_panel_groups! unless dashboard[:panel_groups].is_a?(Array) - - for_panel_groups do |panel_group| - for_panels_in(panel_group) do |panel| - missing_metrics! unless panel[:metrics].is_a?(Array) - - panel[:metrics].each do |metric| - yield metric - end - end - end - end - - def for_variables - return unless dashboard.dig(:templating, :variables).is_a?(Hash) - - dashboard.dig(:templating, :variables).each do |variable_name, variable| - yield variable_name, variable - end - end - - def for_panel_groups - dashboard[:panel_groups].each do |panel_group| - yield panel_group - end - end - - def for_panels_in(panel_group) - missing_panels! unless panel_group[:panels].is_a?(Array) - - panel_group[:panels].each do |panel| - yield panel - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb deleted file mode 100644 index ad9d78133af..00000000000 --- a/lib/gitlab/metrics/dashboard/stages/url_validator.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Stages - class UrlValidator < BaseStage - def transform! - validate_dashboard_links(dashboard) - - validate_chart_links(dashboard) - end - - private - - def blocker_args - { - schemes: %w(http https), - ports: [], - allow_localhost: allow_setting_local_requests?, - allow_local_network: allow_setting_local_requests?, - ascii_only: false, - enforce_user: false, - enforce_sanitization: false, - dns_rebind_protection: true - } - end - - def allow_setting_local_requests? - Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? - end - - def validate_dashboard_links(dashboard) - validate_links(dashboard[:links]) - end - - def validate_chart_links(dashboard) - dashboard[:panel_groups].each do |panel_group| - panel_group[:panels].each do |panel| - validate_links(panel[:links]) - end - end - end - - def validate_links(links) - links&.each do |link| - next unless link.is_a? Hash - - Gitlab::UrlBlocker.validate!(link[:url], **blocker_args) - rescue Gitlab::UrlBlocker::BlockedUrlError - link[:url] = '' - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/transformers/errors.rb b/lib/gitlab/metrics/dashboard/transformers/errors.rb deleted file mode 100644 index bc85dc4e131..00000000000 --- a/lib/gitlab/metrics/dashboard/transformers/errors.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - module Dashboard - module Transformers - module Errors - BaseError = Class.new(StandardError) - - class MissingAttribute < BaseError - def initialize(attribute_name) - super("Missing attribute: '#{attribute_name}'") - end - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb deleted file mode 100644 index e7b901861ef..00000000000 --- a/lib/gitlab/metrics/dashboard/url.rb +++ /dev/null @@ -1,133 +0,0 @@ -# frozen_string_literal: true - -# Manages url matching for metrics dashboards. -module Gitlab - module Metrics - module Dashboard - class Url - class << self - include Gitlab::Utils::StrongMemoize - - QUERY_PATTERN = '(?<query>\?[a-zA-Z0-9%.()+_=-]+(&[a-zA-Z0-9%.()+_=-]+)*)?' - ANCHOR_PATTERN = '(?<anchor>\#[a-z0-9_-]+)?' - DASH_PATTERN = '(?:/-)' - - # Matches dashboard urls for a Grafana embed. - # - # EX - https://<host>/<namespace>/<project>/grafana/metrics_dashboard - def grafana_regex - strong_memoize(:grafana_regex) do - regex_for_project_metrics( - %r{ - #{DASH_PATTERN}? - /grafana - /metrics_dashboard - }xo - ) - end - end - - # Matches dashboard urls for a metric chart embed - # for cluster metrics. - # This regex needs to match the dashboard URL as well, not just the trigger URL. - # The inline_metrics_redactor_filter.rb uses this regex to match against - # the dashboard URL. - # - # EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB) - # dashboard URL - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/metrics_dashboard?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB) - def clusters_regex - strong_memoize(:clusters_regex) do - regex_for_project_metrics( - %r{ - #{DASH_PATTERN}? - /clusters - /(?<cluster_id>\d+) - /? - ( (/metrics) | ( /metrics_dashboard\.json ) )? - }xo - ) - end - end - - # Matches dashboard urls for a metric chart embed - # for a specifc firing GitLab alert - # - # EX - https://<host>/<namespace>/<project>/prometheus/alerts/<alert_id>/metrics_dashboard - def alert_regex - strong_memoize(:alert_regex) do - regex_for_project_metrics( - %r{ - #{DASH_PATTERN}? - /prometheus - /alerts - /(?<alert>\d+) - /metrics_dashboard(\.json)? - }xo - ) - end - end - - # Parses query params out from full url string into hash. - # - # Ex) 'https://<root>/<project>/<environment>/metrics?title=Title&group=Group' - # --> { title: 'Title', group: 'Group' } - def parse_query(url) - query_string = URI.parse(url).query.to_s - - CGI.parse(query_string) - .transform_values { |value| value.first } - .symbolize_keys - end - - private - - def environment_metrics_regex - %r{ - #{DASH_PATTERN}? - /environments - /(?<environment>\d+) - /(metrics_dashboard|metrics) - }xo - end - - def non_environment_metrics_regex - %r{ - #{DASH_PATTERN} - /metrics - (?= # Lookahead to ensure there is an environment query param - \? - .* - environment=(?<environment>\d+) - .* - ) - }xo - end - - def regex_for_project_metrics(path_suffix_pattern) - %r{ - ^(?<url> - #{gitlab_host_pattern} - #{project_path_pattern} - #{path_suffix_pattern} - #{QUERY_PATTERN} - #{ANCHOR_PATTERN} - )$ - }x - end - - def gitlab_host_pattern - Regexp.escape(gitlab_domain) - end - - def project_path_pattern - "\/#{Project.reference_pattern}" - end - - def gitlab_domain - Gitlab.config.gitlab.url - end - end - end - end - end -end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index b4baeba72e8..c299fa37e7a 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -4,13 +4,13 @@ module Gitlab module Metrics class RequestsRackMiddleware HTTP_METHODS = { - "delete" => %w(200 202 204 303 400 401 403 404 500 503), - "get" => %w(200 204 301 302 303 304 307 400 401 403 404 410 422 429 500 503), - "head" => %w(200 204 301 302 303 401 403 404 410 500), - "options" => %w(200 404), - "patch" => %w(200 202 204 400 403 404 409 416 500), - "post" => %w(200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 422 429 500 503), - "put" => %w(200 202 204 400 401 403 404 405 406 409 410 422 500) + "delete" => %w[200 202 204 303 400 401 403 404 500 503], + "get" => %w[200 204 301 302 303 304 307 400 401 403 404 410 422 429 500 503], + "head" => %w[200 204 301 302 303 401 403 404 410 500], + "options" => %w[200 404], + "patch" => %w[200 202 204 400 403 404 409 416 500], + "post" => %w[200 201 202 204 301 302 303 304 400 401 403 404 406 409 410 412 422 429 500 503], + "put" => %w[200 202 204 400 401 403 404 405 406 409 410 422 500] }.freeze HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$}.freeze diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index 86372973c82..8d4f4570c21 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -54,7 +54,7 @@ module Gitlab next if load_balancer.primary_only? load_balancer.host_list.hosts.each do |host| - stats << { labels: labels_for_replica_host(load_balancer, host), stats: host.connection.pool.stat } + stats << { labels: labels_for_replica_host(load_balancer, host), stats: host.pool.stat } end end end @@ -73,7 +73,7 @@ module Gitlab host: host.host, port: host.port, class: load_balancer.configuration.connection_specification_name, - db_config_name: Gitlab::Database.db_config_name(host.connection) + db_config_name: host.pool.db_config.name } end end diff --git a/lib/gitlab/metrics/subscribers/action_view.rb b/lib/gitlab/metrics/subscribers/action_view.rb index bc9032a6942..ce2cc88e035 100644 --- a/lib/gitlab/metrics/subscribers/action_view.rb +++ b/lib/gitlab/metrics/subscribers/action_view.rb @@ -21,7 +21,7 @@ module Gitlab tags = tags_for(event) current_transaction.observe(:gitlab_view_rendering_duration_seconds, event.duration, tags) do docstring 'View rendering time' - label_keys %i(view) + label_keys %i[view] buckets [0.001, 0.01, 0.1, 1, 10.0] with_feature :prometheus_metrics_view_instrumentation end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 10bb358a292..f9749b65888 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -9,15 +9,15 @@ module Gitlab attach_to :active_record - DB_COUNTERS = %i{count write_count cached_count}.freeze + DB_COUNTERS = %i[count write_count cached_count].freeze SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i.freeze SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze - DB_LOAD_BALANCING_ROLES = %i{replica primary}.freeze - DB_LOAD_BALANCING_COUNTERS = %i{count cached_count wal_count wal_cached_count}.freeze - DB_LOAD_BALANCING_DURATIONS = %i{duration_s}.freeze + DB_LOAD_BALANCING_ROLES = %i[replica primary].freeze + DB_LOAD_BALANCING_COUNTERS = %i[count cached_count wal_count wal_cached_count].freeze + DB_LOAD_BALANCING_DURATIONS = %i[duration_s].freeze SQL_WAL_LOCATION_REGEX = /(pg_current_wal_insert_lsn\(\)::text|pg_last_wal_replay_lsn\(\)::text)/.freeze diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 56a310548a7..a57a859acfe 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -7,7 +7,7 @@ module Gitlab include Gitlab::Metrics::Methods # labels that potentially contain sensitive information and will be filtered - FILTERED_LABEL_KEYS = %i(branch path).freeze + FILTERED_LABEL_KEYS = %i[branch path].freeze # The series to store events (e.g. Git pushes) in. EVENT_SERIES = 'events' diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index fcfa86734e8..f3c1e6897af 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -9,7 +9,7 @@ module Gitlab # etc. class WebTransaction < Transaction THREAD_KEY = :_gitlab_metrics_transaction - BASE_LABEL_KEYS = %i(controller action feature_category).freeze + BASE_LABEL_KEYS = %i[controller action feature_category].freeze CONTROLLER_KEY = 'action_controller.instance' ENDPOINT_KEY = 'api.endpoint' diff --git a/lib/gitlab/middleware/read_only/controller.rb b/lib/gitlab/middleware/read_only/controller.rb index 69e2ae55cb0..09a813fa4c3 100644 --- a/lib/gitlab/middleware/read_only/controller.rb +++ b/lib/gitlab/middleware/read_only/controller.rb @@ -4,26 +4,26 @@ module Gitlab module Middleware class ReadOnly class Controller - DISALLOWED_METHODS = %w(POST PATCH PUT DELETE).freeze + DISALLOWED_METHODS = %w[POST PATCH PUT DELETE].freeze APPLICATION_JSON = 'application/json' - APPLICATION_JSON_TYPES = %W{#{APPLICATION_JSON} application/vnd.git-lfs+json}.freeze + APPLICATION_JSON_TYPES = %W[#{APPLICATION_JSON} application/vnd.git-lfs+json].freeze ERROR_MESSAGE = 'You cannot perform write operations on a read-only instance' ALLOWLISTED_GIT_READ_ONLY_ROUTES = { - 'repositories/git_http' => %w{git_upload_pack} + 'repositories/git_http' => %w[git_upload_pack] }.freeze ALLOWLISTED_GIT_LFS_BATCH_ROUTES = { - 'repositories/lfs_api' => %w{batch} + 'repositories/lfs_api' => %w[batch] }.freeze ALLOWLISTED_GIT_REVISION_ROUTES = { - 'projects/compare' => %w{create} + 'projects/compare' => %w[create] }.freeze ALLOWLISTED_SESSION_ROUTES = { - 'sessions' => %w{destroy}, - 'admin/sessions' => %w{create destroy} + 'sessions' => %w[destroy], + 'admin/sessions' => %w[create destroy] }.freeze GRAPHQL_URL = '/api/graphql' diff --git a/lib/gitlab/observability.rb b/lib/gitlab/observability.rb index b500df86363..a4e18cc170b 100644 --- a/lib/gitlab/observability.rb +++ b/lib/gitlab/observability.rb @@ -27,13 +27,8 @@ module Gitlab "#{Gitlab::Observability.observability_url}/v1/auth/start" end - def tracing_url(project) - "#{Gitlab::Observability.observability_url}/query/#{project.group.id}/#{project.id}/v1/traces" - end - - def provisioning_url(_project) - # TODO Change to correct endpoint when API is ready - Gitlab::Observability.observability_url.to_s + def provisioning_url(project) + "#{Gitlab::Observability.observability_url}/v3/tenant/#{project.id}" end # Returns true if the GitLab Observability UI (GOUI) feature flag is enabled @@ -46,11 +41,6 @@ module Gitlab Feature.enabled?(:observability_group_tab) end - # Returns true if Tracing UI is enabled - def tracing_enabled?(project) - Feature.enabled?(:observability_tracing, project) - end - # Returns the embeddable Observability URL of a given URL # # - Validates the URL @@ -163,3 +153,5 @@ module Gitlab end end end + +Gitlab::Observability.prepend_mod_with('Gitlab::Observability') diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb index 4e0e5102bec..defe054d631 100644 --- a/lib/gitlab/pages.rb +++ b/lib/gitlab/pages.rb @@ -5,6 +5,7 @@ module Gitlab VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request' MAX_SIZE = 1.terabyte + DEPLOYMENT_EXPIRATION = 24.hours include JwtAuthenticatable @@ -23,6 +24,29 @@ module Gitlab ::Gitlab.config.pages.access_control && ::Gitlab::CurrentSettings.current_application_settings.force_pages_access_control end + + def enabled? + Gitlab.config.pages.enabled + end + + def add_unique_domain_to(project) + return unless enabled? + # If the project used a unique domain once, it'll always use the same + return if project.project_setting.pages_unique_domain_in_database.present? + + project.project_setting.pages_unique_domain_enabled = true + project.project_setting.pages_unique_domain = Gitlab::Pages::RandomDomain.generate( + project_path: project.path, + namespace_path: project.parent.full_path) + end + + def multiple_versions_enabled_for?(project) + return false if project.blank? + + ::Feature.enabled?(:pages_multiple_versions_setting, project) && + project.licensed_feature_available?(:pages_multiple_versions) && + project.project_setting.pages_multiple_versions_enabled + end end end end diff --git a/lib/gitlab/pages/virtual_host_finder.rb b/lib/gitlab/pages/virtual_host_finder.rb index 88ee0e44c00..e9ac86f0d47 100644 --- a/lib/gitlab/pages/virtual_host_finder.rb +++ b/lib/gitlab/pages/virtual_host_finder.rb @@ -38,15 +38,9 @@ module Gitlab return if namespace.blank? - cache = if Feature.enabled?(:cache_pages_domain_api, namespace) - ::Gitlab::Pages::CacheControl.for_namespace(namespace.id) - end - ::Pages::VirtualDomain.new( trim_prefix: namespace.full_path, - projects: namespace.all_projects_with_pages, - cache: cache - ) + projects: namespace.all_projects_with_pages) end def by_custom_domain(host) @@ -54,15 +48,7 @@ module Gitlab return unless domain&.pages_deployed? - cache = if Feature.enabled?(:cache_pages_domain_api, domain.project.root_namespace) - ::Gitlab::Pages::CacheControl.for_domain(domain.id) - end - - ::Pages::VirtualDomain.new( - projects: [domain.project], - domain: domain, - cache: cache - ) + ::Pages::VirtualDomain.new(projects: [domain.project], domain: domain) end end end diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index ee8259cc671..592f635c14e 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -10,6 +10,20 @@ module Gitlab ::Packages::BuildInfo => { id: :desc } }.freeze + SUPPORTED_MULTI_ORDERING = { + Group => { name: [:asc] }, + AuditEvent => { id: [:desc] }, + User => { + id: [:asc, :desc], + name: [:asc, :desc], + username: [:asc, :desc], + created_at: [:asc, :desc], + updated_at: [:asc, :desc] + }, + ::Ci::Build => { id: [:desc] }, + ::Packages::BuildInfo => { id: [:desc] } + }.freeze + # Relation types that are enforced in this list # enforce the use of keyset pagination, thus erroring out requests # made with offset pagination above a certain limit. @@ -19,7 +33,11 @@ module Gitlab ENFORCED_TYPES = [Group].freeze def self.available_for_type?(relation) - SUPPORTED_ORDERING.key?(relation.klass) + if Feature.enabled?(:api_keyset_pagination_multi_order) + SUPPORTED_MULTI_ORDERING.key?(relation.klass) + else + SUPPORTED_ORDERING.key?(relation.klass) + end end def self.available?(cursor_based_request_context, relation) @@ -32,9 +50,16 @@ module Gitlab end def self.order_satisfied?(relation, cursor_based_request_context) - order_by_from_request = cursor_based_request_context.order_by + if Feature.enabled?(:api_keyset_pagination_multi_order) + order_by_from_request = cursor_based_request_context.order + sort_from_request = cursor_based_request_context.sort + + SUPPORTED_MULTI_ORDERING[relation.klass][order_by_from_request]&.include?(sort_from_request) + else + order_by_from_request = cursor_based_request_context.order_by - SUPPORTED_ORDERING[relation.klass] == order_by_from_request + SUPPORTED_ORDERING[relation.klass] == order_by_from_request + end end private_class_method :order_satisfied? end diff --git a/lib/gitlab/pagination/keyset/cursor_based_request_context.rb b/lib/gitlab/pagination/keyset/cursor_based_request_context.rb index 41b90846345..f90574c86ab 100644 --- a/lib/gitlab/pagination/keyset/cursor_based_request_context.rb +++ b/lib/gitlab/pagination/keyset/cursor_based_request_context.rb @@ -32,6 +32,14 @@ module Gitlab def order_by { (params[:order_by]&.to_sym || DEFAULT_SORT_COLUMN) => (params[:sort]&.to_sym || DEFAULT_SORT_DIRECTION) } end + + def order + params[:order_by]&.to_sym || DEFAULT_SORT_COLUMN + end + + def sort + params[:sort]&.to_sym || DEFAULT_SORT_DIRECTION + end end end end diff --git a/lib/gitlab/patch/redis_cache_store.rb b/lib/gitlab/patch/redis_cache_store.rb index ea6e1f11bc9..96729056ce5 100644 --- a/lib/gitlab/patch/redis_cache_store.rb +++ b/lib/gitlab/patch/redis_cache_store.rb @@ -3,8 +3,6 @@ module Gitlab module Patch module RedisCacheStore - PIPELINE_BATCH_SIZE = 100 - # We will try keep patched code explicit and matching the original signature in # https://github.com/rails/rails/blob/v6.1.7.2/activesupport/lib/active_support/cache/redis_cache_store.rb#L361 def read_multi_mget(*names) # rubocop:disable Style/ArgumentsForwarding @@ -21,7 +19,7 @@ module Gitlab delete_count = 0 redis.with do |conn| - entries.each_slice(PIPELINE_BATCH_SIZE) do |subset| + entries.each_slice(pipeline_batch_size) do |subset| delete_count += Gitlab::Redis::CrossSlot::Pipeline.new(conn).pipelined do |pipeline| subset.each { |entry| pipeline.del(entry) } end.sum @@ -59,7 +57,7 @@ module Gitlab end def pipeline_mget(conn, keys) - keys.each_slice(PIPELINE_BATCH_SIZE).flat_map do |subset| + keys.each_slice(pipeline_batch_size).flat_map do |subset| Gitlab::Redis::CrossSlot::Pipeline.new(conn).pipelined do |p| subset.each { |key| p.get(key) } end @@ -68,6 +66,10 @@ module Gitlab private + def pipeline_batch_size + @pipeline_batch_size ||= [ENV['GITLAB_REDIS_CLUSTER_PIPELINE_BATCH_LIMIT'].to_i, 1000].max + end + def enable_rails_cache_pipeline_patch? redis.with { |c| ::Gitlab::Redis::ClusterUtil.cluster?(c) } end diff --git a/lib/gitlab/patch/sidekiq_scheduled_enq.rb b/lib/gitlab/patch/sidekiq_scheduled_enq.rb new file mode 100644 index 00000000000..de0e8465f97 --- /dev/null +++ b/lib/gitlab/patch/sidekiq_scheduled_enq.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +# Patch to address https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2286 +# Using a dual-namespace poller eliminates the need for script based migration of +# schedule-related sets in Sidekiq. +module Gitlab + module Patch + module SidekiqScheduledEnq + # The patched enqueue_jobs will poll non-namespaced scheduled sets before doing the same for + # namespaced sets via super and vice-versa depending on how Sidekiq.redis was configured + def enqueue_jobs(sorted_sets = Sidekiq::Scheduled::SETS) + # checks the other namespace + if Gitlab::Utils.to_boolean(ENV['SIDEKIQ_ENABLE_DUAL_NAMESPACE_POLLING'], default: true) + # Refer to https://github.com/sidekiq/sidekiq/blob/v6.5.7/lib/sidekiq/scheduled.rb#L25 + # this portion swaps out Sidekiq.redis for Gitlab::Redis::Queues + Gitlab::Redis::Queues.with do |conn| # rubocop:disable Cop/RedisQueueUsage + sorted_sets.each do |sorted_set| + # adds namespace if `super` polls with a non-namespaced Sidekiq.redis + if Gitlab::Utils.to_boolean(ENV['SIDEKIQ_ENQUEUE_NON_NAMESPACED']) + sorted_set = "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{sorted_set}" # rubocop:disable Cop/RedisQueueUsage + end + + while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s])) # rubocop:disable Gitlab/ModuleWithInstanceVariables, Lint/AssignmentInCondition + Sidekiq::Client.push(Sidekiq.load_json(job)) # rubocop:disable Cop/SidekiqApiUsage + Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" } + end + end + end + end + + # calls original enqueue_jobs which may or may not be namespaced depending on SIDEKIQ_ENQUEUE_NON_NAMESPACED + super + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index b4297cc695b..6b198d5d950 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -68,7 +68,7 @@ module Gitlab def limited_notes_count return @limited_notes_count if defined?(@limited_notes_count) - types = %w(issue merge_request commit snippet) + types = %w[issue merge_request commit snippet] @limited_notes_count = 0 types.each do |type| diff --git a/lib/gitlab/prometheus/additional_metrics_parser.rb b/lib/gitlab/prometheus/additional_metrics_parser.rb deleted file mode 100644 index f5eb27b6916..00000000000 --- a/lib/gitlab/prometheus/additional_metrics_parser.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - module AdditionalMetricsParser - CONFIG_ROOT = 'config/prometheus' - MUTEX = Mutex.new - extend self - - def load_groups_from_yaml(file_name) - yaml_metrics_raw(file_name).map(&method(:group_from_entry)) - end - - private - - def validate!(obj) - raise ParsingError, obj.errors.full_messages.join('\n') unless obj.valid? - end - - def group_from_entry(entry) - entry[:name] = entry.delete(:group) - entry[:metrics]&.map! do |entry| - Metric.new(entry).tap(&method(:validate!)) - end - - MetricGroup.new(entry).tap(&method(:validate!)) - end - - def yaml_metrics_raw(file_name) - load_yaml_file(file_name)&.map(&:deep_symbolize_keys).freeze - end - - # rubocop:disable Gitlab/ModuleWithInstanceVariables - def load_yaml_file(file_name) - return YAML.load_file(Rails.root.join(CONFIG_ROOT, file_name)) if Rails.env.development? - - MUTEX.synchronize do - @loaded_yaml_cache ||= {} - @loaded_yaml_cache[file_name] ||= YAML.load_file(Rails.root.join(CONFIG_ROOT, file_name)) - end - end - # rubocop:enable Gitlab/ModuleWithInstanceVariables - end - end -end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb deleted file mode 100644 index ab6ef7d5466..00000000000 --- a/lib/gitlab/prometheus/queries/additional_metrics_deployment_query.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - module Queries - class AdditionalMetricsDeploymentQuery < BaseQuery - include QueryAdditionalMetrics - - # rubocop: disable CodeReuse/ActiveRecord - def query(deployment_id) - Deployment.find_by(id: deployment_id).try do |deployment| - query_metrics( - deployment.project, - deployment.environment, - common_query_context( - deployment.environment, - timeframe_start: (deployment.created_at - 30.minutes).to_f, - timeframe_end: (deployment.created_at + 30.minutes).to_f - ) - ) - end - end - # rubocop: enable CodeReuse/ActiveRecord - end - end - end -end diff --git a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb b/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb deleted file mode 100644 index c49877ddf9d..00000000000 --- a/lib/gitlab/prometheus/queries/additional_metrics_environment_query.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - module Queries - class AdditionalMetricsEnvironmentQuery < BaseQuery - include QueryAdditionalMetrics - - # rubocop: disable CodeReuse/ActiveRecord - def query(environment_id, timeframe_start = 8.hours.ago, timeframe_end = Time.now) - ::Environment.find_by(id: environment_id).try do |environment| - query_metrics( - environment.project, - environment, - common_query_context( - environment, - timeframe_start: timeframe_start.to_f, - timeframe_end: timeframe_end.to_f - ) - ) - end - end - # rubocop: enable CodeReuse/ActiveRecord - end - end - end -end diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb index 5b688f83545..6a6e5c22d63 100644 --- a/lib/gitlab/prometheus/query_variables.rb +++ b/lib/gitlab/prometheus/query_variables.rb @@ -9,7 +9,7 @@ module Gitlab __range: range(start_time, end_time), ci_environment_slug: environment.slug, kube_namespace: environment.deployment_namespace || '', - environment_filter: %{container_name!="POD",environment="#{environment.slug}"}, + environment_filter: %(container_name!="POD",environment="#{environment.slug}"), ci_project_name: environment.project.name, ci_project_namespace: environment.project.namespace.name, ci_project_path: environment.project.full_path, diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index 498da38e268..f44e5383b4f 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -68,7 +68,7 @@ module Gitlab GEO_NODES_LOAD = 'SELECT 1 AS one FROM "geo_nodes" LIMIT 1' LICENSES_LOAD = 'SELECT "licenses".* FROM "licenses" ORDER BY "licenses"."id"' - SCHEMA_INTROSPECTION = %r/SELECT.*(FROM|JOIN) (pg_attribute|pg_class)/m.freeze + SCHEMA_INTROSPECTION = %r{SELECT.*(FROM|JOIN) (pg_attribute|pg_class)}m.freeze # queries can be safely ignored if they are amoritized in regular usage # (i.e. only requested occasionally and otherwise cached). diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index 829b305d1ee..2182f5b56e4 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -81,6 +81,55 @@ module Gitlab user_allowlist end + ThrottleDefinition = Struct.new(:options, :request_identifier) + def self.throttle_definitions + { + 'throttle_unauthenticated_web' => ThrottleDefinition.new( + Gitlab::Throttle.unauthenticated_web_options, + ->(req) { req.ip if req.throttle_unauthenticated_web? } + ), + # Product analytics feature is in experimental stage. + # At this point we want to limit amount of events registered + # per application (aid stands for application id). + 'throttle_product_analytics_collector' => ThrottleDefinition.new( + { limit: 100, period: 60 }, + ->(req) { req.params['aid'] if req.product_analytics_collector_request? } + ), + 'throttle_authenticated_web' => ThrottleDefinition.new( + Gitlab::Throttle.authenticated_web_options, + ->(req) { req.throttled_identifer([:api, :rss, :ics]) if req.throttle_authenticated_web? } + ), + 'throttle_unauthenticated_protected_paths' => ThrottleDefinition.new( + Gitlab::Throttle.protected_paths_options, + ->(req) { req.ip if req.throttle_unauthenticated_protected_paths? } + ), + 'throttle_authenticated_protected_paths_api' => ThrottleDefinition.new( + Gitlab::Throttle.protected_paths_options, + ->(req) { req.throttled_identifer([:api]) if req.throttle_authenticated_protected_paths_api? } + ), + 'throttle_authenticated_protected_paths_web' => ThrottleDefinition.new( + Gitlab::Throttle.protected_paths_options, + ->(req) { req.throttled_identifer([:api, :rss, :ics]) if req.throttle_authenticated_protected_paths_web? } + ), + 'throttle_unauthenticated_get_protected_paths' => ThrottleDefinition.new( + Gitlab::Throttle.protected_paths_options, + ->(req) { req.ip if req.throttle_unauthenticated_get_protected_paths? } + ), + 'throttle_authenticated_get_protected_paths_api' => ThrottleDefinition.new( + Gitlab::Throttle.protected_paths_options, + ->(req) { req.throttled_identifer([:api]) if req.throttle_authenticated_get_protected_paths_api? } + ), + 'throttle_authenticated_get_protected_paths_web' => ThrottleDefinition.new( + Gitlab::Throttle.protected_paths_options, + ->(req) { req.throttled_identifer([:api, :rss, :ics]) if req.throttle_authenticated_get_protected_paths_web? } + ), + 'throttle_authenticated_git_lfs' => ThrottleDefinition.new( + Gitlab::Throttle.throttle_authenticated_git_lfs_options, + ->(req) { req.throttled_identifer([:api]) if req.throttle_authenticated_git_lfs? } + ) + } + end + def self.configure_throttles(rack_attack) # Each of these settings follows the same pattern of specifying separate # authenticated and unauthenticated rates via settings @@ -100,49 +149,8 @@ module Gitlab end end - throttle_or_track(rack_attack, 'throttle_unauthenticated_web', Gitlab::Throttle.unauthenticated_web_options) do |req| - if req.throttle_unauthenticated_web? - req.ip - end - end - - # Product analytics feature is in experimental stage. - # At this point we want to limit amount of events registered - # per application (aid stands for application id). - throttle_or_track(rack_attack, 'throttle_product_analytics_collector', limit: 100, period: 60) do |req| - if req.product_analytics_collector_request? - req.params['aid'] - end - end - - throttle_or_track(rack_attack, 'throttle_authenticated_web', Gitlab::Throttle.authenticated_web_options) do |req| - if req.throttle_authenticated_web? - req.throttled_identifer([:api, :rss, :ics]) - end - end - - throttle_or_track(rack_attack, 'throttle_unauthenticated_protected_paths', Gitlab::Throttle.protected_paths_options) do |req| - if req.throttle_unauthenticated_protected_paths? - req.ip - end - end - - throttle_or_track(rack_attack, 'throttle_authenticated_protected_paths_api', Gitlab::Throttle.protected_paths_options) do |req| - if req.throttle_authenticated_protected_paths_api? - req.throttled_identifer([:api]) - end - end - - throttle_or_track(rack_attack, 'throttle_authenticated_protected_paths_web', Gitlab::Throttle.protected_paths_options) do |req| - if req.throttle_authenticated_protected_paths_web? - req.throttled_identifer([:api, :rss, :ics]) - end - end - - throttle_or_track(rack_attack, 'throttle_authenticated_git_lfs', Gitlab::Throttle.throttle_authenticated_git_lfs_options) do |req| - if req.throttle_authenticated_git_lfs? - req.throttled_identifer([:api]) - end + throttle_definitions.each do |name, definition| + throttle_or_track(rack_attack, name, definition.options, &definition.request_identifier) end rack_attack.safelist('throttle_bypass_header') do |req| diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index d7abacb5b67..a03116f5bb2 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -71,6 +71,10 @@ module Gitlab matches?(protected_paths_regex) end + def get_request_protected_path? + matches?(protected_paths_for_get_request_regex) + end + def throttle?(throttle, authenticated:) fragment = Gitlab::Throttle.throttle_fragment!(throttle, authenticated: authenticated) @@ -133,6 +137,28 @@ module Gitlab Gitlab::Throttle.protected_paths_enabled? end + def throttle_unauthenticated_get_protected_paths? + get? && + !should_be_skipped? && + get_request_protected_path? && + Gitlab::Throttle.protected_paths_enabled? && + unauthenticated? + end + + def throttle_authenticated_get_protected_paths_api? + get? && + api_request? && + get_request_protected_path? && + Gitlab::Throttle.protected_paths_enabled? + end + + def throttle_authenticated_get_protected_paths_web? + get? && + web_request? && + get_request_protected_path? && + Gitlab::Throttle.protected_paths_enabled? + end + def throttle_unauthenticated_packages_api? packages_api_path? && Gitlab::Throttle.settings.throttle_unauthenticated_packages_api_enabled && @@ -199,6 +225,14 @@ module Gitlab Regexp.union(protected_paths.map { |path| /\A#{Regexp.escape(path)}/ }) end + def protected_paths_for_get_request + Gitlab::CurrentSettings.current_application_settings.protected_paths_for_get_request + end + + def protected_paths_for_get_request_regex + Regexp.union(protected_paths_for_get_request.map { |path| /\A#{Regexp.escape(path)}/ }) + end + def packages_api_path? matches?(::Gitlab::Regex::Packages::API_PATH_REGEX) end diff --git a/lib/gitlab/rack_load_balancing_helpers.rb b/lib/gitlab/rack_load_balancing_helpers.rb new file mode 100644 index 00000000000..5d7228a38fd --- /dev/null +++ b/lib/gitlab/rack_load_balancing_helpers.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module RackLoadBalancingHelpers + def load_balancer_stick_request(model, namespace, id) + request.env[::Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT] ||= Set.new + request.env[::Gitlab::Database::LoadBalancing::RackMiddleware::STICK_OBJECT] << [model.sticking, namespace, id] + + model + .sticking + .find_caught_up_replica(namespace, id) + end + end +end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index e760a576253..89ec996488f 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -13,12 +13,15 @@ module Gitlab Gitlab::Redis::DbLoadBalancing, Gitlab::Redis::FeatureFlag, Gitlab::Redis::Queues, + Gitlab::Redis::QueuesMetadata, + Gitlab::Redis::Pubsub, Gitlab::Redis::RateLimiting, Gitlab::Redis::RepositoryCache, Gitlab::Redis::Sessions, Gitlab::Redis::SharedState, Gitlab::Redis::TraceChunks, - Gitlab::Redis::Chat + Gitlab::Redis::Chat, + Gitlab::Redis::Workhorse ].freeze end end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index 7f4d611a490..9ce030c0bbe 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -19,8 +19,12 @@ module Gitlab end class MethodMissingError < StandardError + def initialize(cmd) + @cmd = cmd + end + def message - 'Method missing. Falling back to execute method on the redis default store in Rails.env.production.' + "Method missing #{@cmd}. Falling back to execute method on the redis default store in Rails.env.production." end end @@ -38,6 +42,17 @@ module Gitlab SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i[info].freeze + # _client and without_reconnect are Redis::Client methods which may be called through multistore + REDIS_CLIENT_COMMANDS = %i[ + _client + without_reconnect + ].freeze + + PUBSUB_SUBSCRIBE_COMMANDS = %i[ + subscribe + unsubscribe + ].freeze + READ_COMMANDS = %i[ exists exists? @@ -71,6 +86,7 @@ module Gitlab incr incrby mapped_hmset + publish rpush sadd sadd? @@ -126,7 +142,7 @@ module Gitlab end # rubocop:disable GitlabSecurity/PublicSend - READ_COMMANDS.each do |name| + (READ_COMMANDS + REDIS_CLIENT_COMMANDS + PUBSUB_SUBSCRIBE_COMMANDS).each do |name| define_method(name) do |*args, **kwargs, &block| if use_primary_and_secondary_stores? read_command(name, *args, **kwargs, &block) @@ -246,9 +262,9 @@ module Gitlab def log_method_missing(command_name, *_args) return if SKIP_LOG_METHOD_MISSING_FOR_COMMANDS.include?(command_name) - raise MethodMissingError if Rails.env.test? || Rails.env.development? + raise MethodMissingError, command_name if Rails.env.test? || Rails.env.development? - log_error(MethodMissingError.new, command_name) + log_error(MethodMissingError.new(command_name), command_name) increment_method_missing_count(command_name) end diff --git a/lib/gitlab/redis/pubsub.rb b/lib/gitlab/redis/pubsub.rb new file mode 100644 index 00000000000..b5022f467a2 --- /dev/null +++ b/lib/gitlab/redis/pubsub.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class Pubsub < ::Gitlab::Redis::Wrapper + class << self + def config_fallback + SharedState + end + end + end + end +end diff --git a/lib/gitlab/redis/etag_cache.rb b/lib/gitlab/redis/queues_metadata.rb index 6aafdc8e518..bb83e7709e1 100644 --- a/lib/gitlab/redis/etag_cache.rb +++ b/lib/gitlab/redis/queues_metadata.rb @@ -2,17 +2,17 @@ module Gitlab module Redis - class EtagCache < ::Gitlab::Redis::Wrapper + class QueuesMetadata < ::Gitlab::Redis::Wrapper class << self - def store_name - 'Cache' + def config_fallback + Queues end private def redis - primary_store = ::Redis.new(Gitlab::Redis::Cache.params) - secondary_store = ::Redis.new(Gitlab::Redis::SharedState.params) + primary_store = ::Redis.new(params) + secondary_store = ::Redis.new(config_fallback.params) MultiStore.new(primary_store, secondary_store, name.demodulize) end diff --git a/lib/gitlab/redis/workhorse.rb b/lib/gitlab/redis/workhorse.rb new file mode 100644 index 00000000000..ea0fca515fe --- /dev/null +++ b/lib/gitlab/redis/workhorse.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class Workhorse < ::Gitlab::Redis::Wrapper + class << self + def config_fallback + SharedState + end + + private + + def redis + primary_store = ::Redis.new(params) + secondary_store = ::Redis.new(config_fallback.params) + + MultiStore.new(primary_store, secondary_store, store_name) + end + end + end + end +end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 45fe04835cc..1ec8818d3f5 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -16,6 +16,8 @@ require 'redis/store/factory' module Gitlab module Redis class Wrapper + InvalidPathError = Class.new(StandardError) + class << self delegate :params, :url, :store, to: :new @@ -122,12 +124,14 @@ module Gitlab config = raw_config_hash config[:instrumentation_class] ||= self.class.instrumentation_class - if config[:cluster].present? - config[:db] = 0 # Redis Cluster only supports db 0 - config - else - parse_redis_url(config) - end + result = if config[:cluster].present? + config[:db] = 0 # Redis Cluster only supports db 0 + config + else + parse_redis_url(config) + end + + parse_client_tls_options(result) end def parse_redis_url(config) @@ -153,6 +157,33 @@ module Gitlab end end + def parse_client_tls_options(config) + return config unless config&.key?(:ssl_params) + + # Only cert_file and key_file are handled in this method. ca_file and + # ca_path are Strings, so they can be passed as-is. cert_store is not + # currently supported. + + cert_file = config[:ssl_params].delete(:cert_file) + key_file = config[:ssl_params].delete(:key_file) + + unless ::File.exist?(cert_file) + raise InvalidPathError, + "Certificate file #{cert_file} specified in in `resque.yml` does not exist." + end + + config[:ssl_params][:cert] = OpenSSL::X509::Certificate.new(File.read(cert_file)) + + unless ::File.exist?(key_file) + raise InvalidPathError, + "Key file #{key_file} specified in in `resque.yml` does not exist." + end + + config[:ssl_params][:key] = OpenSSL::PKey.read(File.read(key_file)) + + config + end + def raw_config_hash config_data = fetch_config diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 783b68fac12..a5e3806735f 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -3,9 +3,9 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor - REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project + REFERABLES = %i[user issue label milestone mentioned_user mentioned_group mentioned_project merge_request snippet commit commit_range directly_addressed_user epic vulnerability - alert).freeze + alert].freeze attr_accessor :project, :current_user, :author def initialize(project, current_user = nil) @@ -43,7 +43,7 @@ module Gitlab @references[type] ||= references(type) end - next unless %w(mentioned_user mentioned_group mentioned_project).include?(type.to_s) + next unless %w[mentioned_user mentioned_group mentioned_project].include?(type.to_s) define_method("#{type}_ids") do @references[type] ||= references(type, ids_only: true) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 40facabff23..8ef455efe07 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -3,7 +3,6 @@ module Gitlab module Regex extend self - extend BulkImports extend MergeRequests extend Packages @@ -13,7 +12,7 @@ module Gitlab # contains only alphanumeric characters, periods, and underscores, # does not end with a period or forward slash, and has no leading or trailing forward slashes # eg 'destination-path' or 'destination_pth' not 'example/com/destination/full/path' - @group_path_regex ||= %r/\A[.]?[^\W]([.]?[0-9a-z][-_]*)+\z/i + @group_path_regex ||= %r{\A[.]?[^\W]([.]?[0-9a-z][-_]*)+\z}i end def group_path_regex_message @@ -107,7 +106,7 @@ module Gitlab end def cluster_agent_name_regex_message - %q{can contain only lowercase letters, digits, and '-', but cannot start or end with '-'} + %q(can contain only lowercase letters, digits, and '-', but cannot start or end with '-') end def kubernetes_namespace_regex @@ -314,7 +313,7 @@ module Gitlab # One or more `part`s, separated by separator def sep_by_1(separator, part) - %r(#{part} (#{separator} #{part})*)x + %r{#{part} (#{separator} #{part})*}x end def x509_subject_key_identifier_regex diff --git a/lib/gitlab/regex/bulk_imports.rb b/lib/gitlab/regex/bulk_imports.rb deleted file mode 100644 index 65c23b9d2e6..00000000000 --- a/lib/gitlab/regex/bulk_imports.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Regex - module BulkImports - def bulk_import_source_full_path_regex_message - bulk_import_destination_namespace_path_regex_message - end - - def bulk_import_destination_namespace_path_regex_message - "must have a relative path structure " \ - "with no HTTP protocol characters, or leading or trailing forward slashes. " \ - "Path segments must not start or end with a special character, " \ - "and must not contain consecutive special characters." - end - end - end -end diff --git a/lib/gitlab/repo_path.rb b/lib/gitlab/repo_path.rb index a4d1adf7671..f91dccfec7c 100644 --- a/lib/gitlab/repo_path.rb +++ b/lib/gitlab/repo_path.rb @@ -54,6 +54,10 @@ module Gitlab wiki = find_wiki(full_path) [wiki, wiki.try(:project)] + elsif type.design? + design_management_repository = find_design_management_repository(full_path) + + [design_management_repository, design_management_repository.project] else project = find_project(full_path) @@ -95,6 +99,10 @@ module Gitlab container&.try(:wiki) end + def self.find_design_management_repository(full_path) + find_project(full_path)&.design_management_repository + end + def self.extract_snippet_info(snippet_path) path_segments = snippet_path.split('/') snippet_id = path_segments.pop diff --git a/lib/gitlab/saas.rb b/lib/gitlab/saas.rb index 722475ce61d..754eed4f253 100644 --- a/lib/gitlab/saas.rb +++ b/lib/gitlab/saas.rb @@ -18,7 +18,7 @@ module Gitlab end def self.subdomain_regex - %r{\Ahttps://[a-z0-9-]+\.gitlab\.com\z}.freeze + %r{\Ahttps://[a-z0-9-]+\.gitlab\.com\z} end def self.dev_url diff --git a/lib/gitlab/sanitizers/exception_message.rb b/lib/gitlab/sanitizers/exception_message.rb index 11c91093d88..27885c27ee0 100644 --- a/lib/gitlab/sanitizers/exception_message.rb +++ b/lib/gitlab/sanitizers/exception_message.rb @@ -4,8 +4,8 @@ module Gitlab module Sanitizers module ExceptionMessage FILTERED_STRING = '[FILTERED]' - EXCEPTION_NAMES = %w(URI::InvalidURIError Addressable::URI::InvalidURIError).freeze - MESSAGE_REGEX = %r{(\A[^:]+:\s).*\Z}.freeze + EXCEPTION_NAMES = %w[URI::InvalidURIError Addressable::URI::InvalidURIError].freeze + MESSAGE_REGEX = %r{(\A[^:]+:\s).*\Z} class << self def clean(exception_name, message) diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index e302729df66..93deb97c841 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -4,7 +4,7 @@ module Gitlab module Sanitizers class Exif # these tags are not removed from the image - WHITELISTED_TAGS = %w( + WHITELISTED_TAGS = %w[ ResolutionUnit XResolution YResolution @@ -17,13 +17,13 @@ module Gitlab Copyright CopyrightNotice Orientation - ).freeze + ].freeze # these tags are common in exiftool output, these # do not contain any sensitive information, but # we don't need to preserve them when removing # exif tags - IGNORED_TAGS = %w( + IGNORED_TAGS = %w[ ColorComponents EncodingProcess ExifByteOrder @@ -41,11 +41,11 @@ module Gitlab FileType FileTypeExtension MIMEType - ).freeze + ].freeze ALLOWED_TAGS = WHITELISTED_TAGS + IGNORED_TAGS EXCLUDE_PARAMS = WHITELISTED_TAGS.map { |tag| "-#{tag}" } - ALLOWED_MIME_TYPES = %w(image/jpeg image/tiff).freeze + ALLOWED_MIME_TYPES = %w[image/jpeg image/tiff].freeze attr_reader :logger diff --git a/lib/gitlab/sanitizers/svg.rb b/lib/gitlab/sanitizers/svg.rb index 98f78c5e74b..0d4e6be2129 100644 --- a/lib/gitlab/sanitizers/svg.rb +++ b/lib/gitlab/sanitizers/svg.rb @@ -9,7 +9,7 @@ module Gitlab class Scrubber < Loofah::Scrubber # http://www.whatwg.org/specs/web-apps/current-work/multipage/elements.html#embedding-custom-non-visible-data-with-the-data-*-attributes - DATA_ATTR_PATTERN = /\Adata-(?!xml)[a-z_][\w.\u00E0-\u00F6\u00F8-\u017F\u01DD-\u02AF-]*\z/u.freeze + DATA_ATTR_PATTERN = /\Adata-(?!xml)[a-z_][\w.\u00E0-\u00F6\u00F8-\u017F\u01DD-\u02AF-]*\z/u def scrub(node) unless Whitelist::ALLOWED_ELEMENTS.include?(node.name) diff --git a/lib/gitlab/search/abuse_detection.rb b/lib/gitlab/search/abuse_detection.rb index 8711d078ea9..1e4169f3fd7 100644 --- a/lib/gitlab/search/abuse_detection.rb +++ b/lib/gitlab/search/abuse_detection.rb @@ -7,9 +7,9 @@ module Gitlab include AbuseValidators ABUSIVE_TERM_SIZE = 100 - ALLOWED_CHARS_REGEX = %r{\A[[:alnum:]_\-\/\.!]+\z}.freeze + ALLOWED_CHARS_REGEX = %r{\A[[:alnum:]_\-\/\.!]+\z} - ALLOWED_SCOPES = %w( + ALLOWED_SCOPES = %w[ blobs code commits @@ -22,20 +22,20 @@ module Gitlab snippet_titles users wiki_blobs - ).freeze + ].freeze - READABLE_PARAMS = %i( + READABLE_PARAMS = %i[ group_id project_id project_ref query_string repository_ref scope - ).freeze + ].freeze - STOP_WORDS = %w( + STOP_WORDS = %w[ a an and are as at be but by for if in into is it no not of on or such that the their then there these they this to was will with - ).freeze + ].freeze validates :project_id, :group_id, numericality: { only_integer: true, message: "abusive ID detected" }, allow_blank: true diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb index c9051b6a5ff..eb4d61977ea 100644 --- a/lib/gitlab/search/found_blob.rb +++ b/lib/gitlab/search/found_blob.rb @@ -11,8 +11,8 @@ module Gitlab attr_reader :project, :content_match, :blob_path, :highlight_line, :matched_lines_count, :group_level_blob, :group - PATH_REGEXP = /\A(?<ref>[^:]*):(?<path>[^\x00]*)\x00/.freeze - CONTENT_REGEXP = /^(?<ref>[^:]*):(?<path>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze + PATH_REGEXP = /\A(?<ref>[^:]*):(?<path>[^\x00]*)\x00/ + CONTENT_REGEXP = /^(?<ref>[^:]*):(?<path>[^\x00]*)\x00(?<startline>\d+)\x00/ def self.preload_blobs(blobs) to_fetch = blobs.select { |blob| blob.is_a?(self) && blob.blob_path } diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb index 360bbc073c5..bafe7876acd 100644 --- a/lib/gitlab/search/query.rb +++ b/lib/gitlab/search/query.rb @@ -5,8 +5,8 @@ module Gitlab class Query < SimpleDelegator include EncodingHelper - QUOTES_REGEXP = %r{\A"|"\Z}.freeze - TOKEN_WITH_QUOTES_REGEXP = %r{\s(?=(?:[^"]|"[^"]*")*$)}.freeze + QUOTES_REGEXP = %r{\A"|"\Z} + TOKEN_WITH_QUOTES_REGEXP = %r{\s(?=(?:[^"]|"[^"]*")*$)} def initialize(query, filter_opts = {}, &block) @raw_query = query.dup diff --git a/lib/gitlab/search_context.rb b/lib/gitlab/search_context.rb index 04ef2be87f8..9a7f5fc7328 100644 --- a/lib/gitlab/search_context.rb +++ b/lib/gitlab/search_context.rb @@ -130,7 +130,7 @@ module Gitlab elsif view_context.current_controller?(:commits) 'commits' elsif view_context.current_controller?(:groups) - if %w(issues merge_requests).include?(view_context.controller.action_name) + if %w[issues merge_requests].include?(view_context.controller.action_name) view_context.controller.action_name end end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 0e419d0162c..35e01101b3b 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -181,10 +181,7 @@ module Gitlab def projects scope = limit_projects - - if Feature.enabled?(:search_projects_hide_archived, current_user) && !filters[:include_archived] - scope = scope.non_archived - end + scope = scope.non_archived unless filters[:include_archived] scope.search(query) end @@ -193,8 +190,13 @@ module Gitlab issues = IssuesFinder.new(current_user, issuable_params.merge(finder_params)).execute unless default_project_filter - issues = issues.in_projects(project_ids_relation) - .allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/420046") + project_ids = project_ids_relation + if Feature.enabled?(:search_issues_hide_archived_projects, current_user) && !filters[:include_archived] + project_ids = project_ids.non_archived + end + + issues = issues.in_projects(project_ids) + .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/420046') end apply_sort(issues, scope: 'issues') @@ -233,22 +235,21 @@ module Gitlab # Filter milestones by authorized projects. # For performance reasons project_id is being plucked # to be used on a smaller query. - # - # rubocop: disable CodeReuse/ActiveRecord def filter_milestones_by_project(milestones) - project_ids = - milestones.where(project_id: project_ids_relation) - .select(:project_id).distinct - .pluck(:project_id) + candidate_project_ids = project_ids_relation + + if Feature.enabled?(:search_milestones_hide_archived_projects, current_user) && !filters[:include_archived] + candidate_project_ids = candidate_project_ids.non_archived + end + + project_ids = milestones.of_projects(candidate_project_ids).select(:project_id).distinct.pluck(:project_id) # rubocop: disable CodeReuse/ActiveRecord return Milestone.none if project_ids.nil? - authorized_project_ids_relation = - Project.where(id: project_ids).ids_with_issuables_available_for(current_user) + authorized_project_ids_relation = Project.id_in(project_ids).ids_with_issuables_available_for(current_user) - milestones.where(project_id: authorized_project_ids_relation) + milestones.of_projects(authorized_project_ids_relation) end - # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def project_ids_relation diff --git a/lib/gitlab/security/scan_configuration.rb b/lib/gitlab/security/scan_configuration.rb index 9b09ccdeb8e..18767dd332a 100644 --- a/lib/gitlab/security/scan_configuration.rb +++ b/lib/gitlab/security/scan_configuration.rb @@ -33,6 +33,10 @@ module Gitlab def meta_info_path; end + def on_demand_available? + false + end + private attr_reader :project, :configured diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 2e09a4fce12..8f737977e80 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -38,11 +38,11 @@ module Gitlab end def redis_url - Gitlab::Redis::SharedState.url + Gitlab::Redis::Workhorse.url end def redis_db - Gitlab::Redis::SharedState.params.fetch(:db, 0) + Gitlab::Redis::Workhorse.params.fetch(:db, 0) end def get_config_path(dir, _) diff --git a/lib/gitlab/sidekiq_config/worker_matcher.rb b/lib/gitlab/sidekiq_config/worker_matcher.rb index d615d5ecba4..e6b628da0e6 100644 --- a/lib/gitlab/sidekiq_config/worker_matcher.rb +++ b/lib/gitlab/sidekiq_config/worker_matcher.rb @@ -7,7 +7,7 @@ module Gitlab QUERY_OR_OPERATOR = '|' QUERY_AND_OPERATOR = '&' QUERY_CONCATENATE_OPERATOR = ',' - QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze + QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)} QUERY_PREDICATES = { worker_name: :to_s, diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 46939d70c9e..7cc57f9497f 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -257,7 +257,12 @@ module Gitlab end def with_redis(&block) - Gitlab::Redis::Queues.with(&block) # rubocop:disable Cop/RedisQueueUsage, CodeReuse/ActiveRecord + if Feature.enabled?(:use_primary_and_secondary_stores_for_queues_metadata) || + Feature.enabled?(:use_primary_store_as_default_for_queues_metadata) + Gitlab::Redis::QueuesMetadata.with(&block) # rubocop:disable CodeReuse/ActiveRecord + else + Gitlab::Redis::Queues.with(&block) # rubocop:disable Cop/RedisQueueUsage, CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/silent_mode.rb b/lib/gitlab/silent_mode.rb index 7c7cbf8f1d9..f33654c5545 100644 --- a/lib/gitlab/silent_mode.rb +++ b/lib/gitlab/silent_mode.rb @@ -6,6 +6,10 @@ module Gitlab Gitlab::CurrentSettings.silent_mode_enabled? end + def self.enable! + Gitlab::CurrentSettings.update!(silent_mode_enabled: true) + end + def self.log_info(data) Gitlab::AppJsonLogger.info(**add_silent_mode_log_data(data)) end diff --git a/lib/gitlab/slash_commands/deploy.rb b/lib/gitlab/slash_commands/deploy.rb index 16a4875be91..ee2747c0732 100644 --- a/lib/gitlab/slash_commands/deploy.rb +++ b/lib/gitlab/slash_commands/deploy.rb @@ -3,7 +3,7 @@ module Gitlab module SlashCommands class Deploy < BaseCommand - DEPLOY_REGEX = /\Adeploy\s/.freeze + DEPLOY_REGEX = /\Adeploy\s/ def self.match(text) return unless text&.match?(DEPLOY_REGEX) diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index 55497c5e365..9baae1cb627 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -111,7 +111,7 @@ module Gitlab end def fields_with_markdown - %i(title pretext fields) + %i[title pretext fields] end end end diff --git a/lib/gitlab/slug/path.rb b/lib/gitlab/slug/path.rb index 434f36829a6..893fccb807e 100644 --- a/lib/gitlab/slug/path.rb +++ b/lib/gitlab/slug/path.rb @@ -3,10 +3,10 @@ module Gitlab module Slug class Path - LEADING_DASHES = /\A-+/.freeze + LEADING_DASHES = /\A-+/ # Eextract local email part if given an email. Will remove @ sign and everything following it. - EXTRACT_LOCAL_EMAIL_PART = /@.*\z/.freeze - FORBIDDEN_CHARACTERS = /[^a-zA-Z0-9_\-.]/.freeze + EXTRACT_LOCAL_EMAIL_PART = /@.*\z/ + FORBIDDEN_CHARACTERS = /[^a-zA-Z0-9_\-.]/ PATH_TRAILING_VIOLATIONS = %w[.git .atom .].freeze DEFAULT_SLUG = 'blank' diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index d13c3be0a09..d92f8c8df7a 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -15,7 +15,7 @@ module Gitlab update: ::Spamcheck::Action::UPDATE }.freeze - URL_SCHEME_REGEX = %r{^grpc://|^tls://}.freeze + URL_SCHEME_REGEX = %r{^grpc://|^tls://} def initialize @endpoint_url = Gitlab::CurrentSettings.current_application_settings.spam_check_endpoint_url diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index 6563968f315..9fedb2d174f 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -6,15 +6,31 @@ module Gitlab extend ActiveSupport::Concern MIN_CHARS_FOR_PARTIAL_MATCHING = 3 - REGEX_QUOTED_TERM = /(?<=\A| )"[^"]+"(?= |\z)/.freeze + REGEX_QUOTED_TERM = /(?<=\A| )"[^"]+"(?= |\z)/ class_methods do - def fuzzy_search(query, columns, use_minimum_char_limit: true) + def fuzzy_search(query, columns, use_minimum_char_limit: true, exact_matches_first: false) matches = columns.map do |col| fuzzy_arel_match(col, query, use_minimum_char_limit: use_minimum_char_limit) end.compact.reduce(:or) - where(matches) + matches = where(matches) + + return matches unless exact_matches_first + + matches.order(exact_matches_first_sql(query, columns)) + end + + def exact_matches_first_sql(query, columns) + cases_sql = columns.map do |column| + arel_column = column.is_a?(Arel::Attributes::Attribute) ? column : arel_table[column] + match_sql = arel_column.matches(sanitize_sql_like(query)).to_sql + "WHEN #{match_sql} THEN 1" + end + + cases_sql << "ELSE 2" + + Arel.sql("CASE\n#{cases_sql.join("\n")}\nEND") end def to_pattern(query, use_minimum_char_limit: true) diff --git a/lib/gitlab/ssh_public_key.rb b/lib/gitlab/ssh_public_key.rb index 707f7f3fc0a..2b9bfa526cb 100644 --- a/lib/gitlab/ssh_public_key.rb +++ b/lib/gitlab/ssh_public_key.rb @@ -9,12 +9,12 @@ module Gitlab # See https://man.openbsd.org/sshd#AUTHORIZED_KEYS_FILE_FORMAT for the list of # supported algorithms. TECHNOLOGIES = [ - Technology.new(:rsa, SSHData::PublicKey::RSA, [1024, 2048, 3072, 4096], %w(ssh-rsa)), - Technology.new(:dsa, SSHData::PublicKey::DSA, [1024, 2048, 3072], %w(ssh-dss)), - Technology.new(:ecdsa, SSHData::PublicKey::ECDSA, [256, 384, 521], %w(ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521)), - Technology.new(:ed25519, SSHData::PublicKey::ED25519, [256], %w(ssh-ed25519)), - Technology.new(:ecdsa_sk, SSHData::PublicKey::SKECDSA, [256], %w(sk-ecdsa-sha2-nistp256@openssh.com)), - Technology.new(:ed25519_sk, SSHData::PublicKey::SKED25519, [256], %w(sk-ssh-ed25519@openssh.com)) + Technology.new(:rsa, SSHData::PublicKey::RSA, [1024, 2048, 3072, 4096], %w[ssh-rsa]), + Technology.new(:dsa, SSHData::PublicKey::DSA, [1024, 2048, 3072], %w[ssh-dss]), + Technology.new(:ecdsa, SSHData::PublicKey::ECDSA, [256, 384, 521], %w[ecdsa-sha2-nistp256 ecdsa-sha2-nistp384 ecdsa-sha2-nistp521]), + Technology.new(:ed25519, SSHData::PublicKey::ED25519, [256], %w[ssh-ed25519]), + Technology.new(:ecdsa_sk, SSHData::PublicKey::SKECDSA, [256], %w[sk-ecdsa-sha2-nistp256@openssh.com]), + Technology.new(:ed25519_sk, SSHData::PublicKey::SKED25519, [256], %w[sk-ssh-ed25519@openssh.com]) ].freeze def self.technologies diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index f756d229ba1..78c0f04e07e 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -26,7 +26,7 @@ module Gitlab def ask_to_continue return if Gitlab::Utils.to_boolean(ENV['GITLAB_ASSUME_YES']) - answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w{yes no}) + answer = prompt("Do you want to continue (yes/no)? ".color(:blue), %w[yes no]) raise Gitlab::TaskAbortedByUserError unless answer == "yes" end @@ -35,7 +35,7 @@ module Gitlab # It will primarily use lsb_relase to determine the OS. # It has fallbacks to Debian, SuSE, OS X and systems running systemd. def os_name - os_name = run_command(%w(lsb_release -irs)) + os_name = run_command(%w[lsb_release -irs]) os_name ||= if File.readable?('/etc/system-release') File.read('/etc/system-release') @@ -43,7 +43,7 @@ module Gitlab "Debian #{File.read('/etc/debian_version')}" elsif File.readable?('/etc/SuSE-release') File.read('/etc/SuSE-release') - elsif os_x_version = run_command(%w(sw_vers -productVersion)) + elsif os_x_version = run_command(%w[sw_vers -productVersion]) "Mac OS X #{os_x_version}" elsif File.readable?('/etc/os-release') File.read('/etc/os-release').match(/PRETTY_NAME=\"(.+)\"/)[1] @@ -115,7 +115,7 @@ module Gitlab end def uid_for(user_name) - run_command(%W(id -u #{user_name})).chomp.to_i + run_command(%W[id -u #{user_name}]).chomp.to_i end def gid_for(group_name) @@ -130,7 +130,7 @@ module Gitlab def gitlab_user? strong_memoize(:is_gitlab_user) do - current_user = run_command(%w(whoami)).chomp + current_user = run_command(%w[whoami]).chomp current_user == gitlab_user end end @@ -139,7 +139,7 @@ module Gitlab return if gitlab_user? strong_memoize(:warned_user_not_gitlab) do - current_user = run_command(%w(whoami)).chomp + current_user = run_command(%w[whoami]).chomp puts " Warning ".color(:black).background(:yellow) puts " You are running as user #{current_user.color(:magenta)}, we hope you know what you are doing." diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index ec2ec0d801f..102104644f7 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -7,7 +7,7 @@ module Gitlab extend self # Theme ID used when no `default_theme` configuration setting is provided. - APPLICATION_DEFAULT = 1 + APPLICATION_DEFAULT = 3 # Struct class representing a single Theme Theme = Struct.new(:id, :name, :css_class, :css_filename, :primary_color) @@ -15,17 +15,17 @@ module Gitlab # All available Themes def available_themes [ - Theme.new(1, s_('NavigationTheme|Indigo'), 'ui-indigo', 'theme_indigo', '#292961'), - Theme.new(6, s_('NavigationTheme|Light Indigo'), 'ui-light-indigo', 'theme_light_indigo', '#4b4ba3'), - Theme.new(4, s_('NavigationTheme|Blue'), 'ui-blue', 'theme_blue', '#1a3652'), - Theme.new(7, s_('NavigationTheme|Light Blue'), 'ui-light-blue', 'theme_light_blue', '#2261a1'), - Theme.new(5, s_('NavigationTheme|Green'), 'ui-green', 'theme_green', '#0d4524'), - Theme.new(8, s_('NavigationTheme|Light Green'), 'ui-light-green', 'theme_light_green', '#156b39'), - Theme.new(9, s_('NavigationTheme|Red'), 'ui-red', 'theme_red', '#691a16'), - Theme.new(10, s_('NavigationTheme|Light Red'), 'ui-light-red', 'theme_light_red', '#a62e21'), - Theme.new(2, s_('NavigationTheme|Gray'), 'ui-gray', 'theme_gray', '#303030'), - Theme.new(3, s_('NavigationTheme|Light Gray'), 'ui-light-gray', 'theme_light_gray', '#666'), - Theme.new(11, s_('NavigationTheme|Dark Mode (alpha)'), 'gl-dark', nil, '#303030') + Theme.new(1, s_('NavigationTheme|Indigo'), 'ui-indigo', 'theme_indigo', '#222261'), + Theme.new(6, s_('NavigationTheme|Light Indigo'), 'ui-light-indigo', 'theme_light_indigo', '#41419f'), + Theme.new(4, s_('NavigationTheme|Blue'), 'ui-blue', 'theme_blue', '#0b2640'), + Theme.new(7, s_('NavigationTheme|Light Blue'), 'ui-light-blue', 'theme_light_blue', '#145aa1'), + Theme.new(5, s_('NavigationTheme|Green'), 'ui-green', 'theme_green', '#0e4328'), + Theme.new(8, s_('NavigationTheme|Light Green'), 'ui-light-green', 'theme_light_green', '#1b653f'), + Theme.new(9, s_('NavigationTheme|Red'), 'ui-red', 'theme_red', '#580d02'), + Theme.new(10, s_('NavigationTheme|Light Red'), 'ui-light-red', 'theme_light_red', '#a02e1c'), + Theme.new(2, s_('NavigationTheme|Gray'), 'ui-gray', 'theme_gray', '#333238'), + Theme.new(3, s_('NavigationTheme|Light Gray'), 'ui-light-gray', 'theme_light_gray', '#ececef'), + Theme.new(11, s_('NavigationTheme|Dark Mode (alpha)'), 'gl-dark', nil, '#1f1e24') ] end diff --git a/lib/gitlab/time_tracking_formatter.rb b/lib/gitlab/time_tracking_formatter.rb index 7094db14c5d..26efb3b918d 100644 --- a/lib/gitlab/time_tracking_formatter.rb +++ b/lib/gitlab/time_tracking_formatter.rb @@ -8,6 +8,8 @@ module Gitlab CUSTOM_DAY_AND_MONTH_LENGTH = { hours_per_day: 8, days_per_month: 20 }.freeze def parse(string, keep_zero: false) + return unless string + negative_time = string.start_with?('-') string = string.delete_prefix('-') @@ -15,7 +17,10 @@ module Gitlab begin ChronicDuration.parse( string, - CUSTOM_DAY_AND_MONTH_LENGTH.merge(default_unit: 'hours', keep_zero: keep_zero)) + CUSTOM_DAY_AND_MONTH_LENGTH.merge( + default_unit: 'hours', keep_zero: keep_zero, + use_complete_matcher: true + )) rescue StandardError nil end diff --git a/lib/gitlab/tracking/destinations/database_events_snowplow.rb b/lib/gitlab/tracking/destinations/database_events_snowplow.rb index e3512bc4916..458d7f0c129 100644 --- a/lib/gitlab/tracking/destinations/database_events_snowplow.rb +++ b/lib/gitlab/tracking/destinations/database_events_snowplow.rb @@ -16,7 +16,7 @@ module Gitlab override :hostname def hostname - return HOSTNAME if ::Gitlab.com? + return Gitlab::CurrentSettings.snowplow_database_collector_hostname || HOSTNAME if ::Gitlab.com? 'localhost:9091' end diff --git a/lib/gitlab/tracking/service_ping_context.rb b/lib/gitlab/tracking/service_ping_context.rb index d31ca69a10c..2135101e0e3 100644 --- a/lib/gitlab/tracking/service_ping_context.rb +++ b/lib/gitlab/tracking/service_ping_context.rb @@ -3,19 +3,18 @@ module Gitlab module Tracking class ServicePingContext - SCHEMA_URL = 'iglu:com.gitlab/gitlab_service_ping/jsonschema/1-0-0' + SCHEMA_URL = 'iglu:com.gitlab/gitlab_service_ping/jsonschema/1-0-1' REDISHLL_SOURCE = :redis_hll REDIS_SOURCE = :redis ALLOWED_SOURCES = [REDISHLL_SOURCE, REDIS_SOURCE].freeze - def initialize(data_source:, event: nil, key_path: nil) - check_configuration(data_source, event, key_path) + def initialize(data_source:, event: nil) + check_configuration(data_source, event) @payload = { data_source: data_source } - payload[:event_name] = event if data_source.eql? REDISHLL_SOURCE - payload[:key_path] = key_path if data_source.eql? REDIS_SOURCE + payload[:event_name] = event end def to_context @@ -33,18 +32,14 @@ module Gitlab attr_reader :payload - def check_configuration(data_source, event, key_path) - unless ALLOWED_SOURCES.include?(data_source) + def check_configuration(data_source, event) + unless ALLOWED_SOURCES.include?(data_source.to_sym) configuration_error("#{data_source} is not acceptable data source for ServicePingContext") end - if REDISHLL_SOURCE.eql?(data_source) && event.nil? - configuration_error("event attribute can not be missing for #{REDISHLL_SOURCE} data source") - end - - return unless REDIS_SOURCE.eql?(data_source) && key_path.nil? + return unless event.nil? - configuration_error("key_path attribute can not be missing for #{REDIS_SOURCE} data source") + configuration_error("event attribute is required") end def configuration_error(message) diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 61d6fdc6dca..4c04fa7d525 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -50,6 +50,7 @@ module Gitlab plan: plan_name, extra: extra, user_id: user_id, + is_gitlab_team_member: gitlab_team_member?(user_id), namespace_id: namespace_id, project_id: project_id, context_generated_at: Time.current @@ -63,6 +64,13 @@ module Gitlab " Should be one of #{allowed_classes.map(&:to_s)}" Gitlab::ErrorTracking.track_and_raise_for_dev_exception(ArgumentError.new(exception)) end + + # Overridden in EE + def gitlab_team_member?(_user_id) + nil + end end end end + +Gitlab::Tracking::StandardContext.prepend_mod diff --git a/lib/gitlab/unicode.rb b/lib/gitlab/unicode.rb index f291ea1b4ee..b78393dd101 100644 --- a/lib/gitlab/unicode.rb +++ b/lib/gitlab/unicode.rb @@ -7,13 +7,13 @@ module Gitlab # # Documentation on how this works: # https://idiosyncratic-ruby.com/41-proper-unicoding.html - BIDI_REGEXP = /\p{Bidi Control}/.freeze + BIDI_REGEXP = /\p{Bidi Control}/ # Regular expression for identifying space characters # # In web browsers space characters can be confused with simple # spaces which may be misleading - SPACE_REGEXP = /\p{Space_Separator}/.freeze + SPACE_REGEXP = /\p{Space_Separator}/ class << self # Warning message used to highlight bidi characters in the GUI diff --git a/lib/gitlab/untrusted_regexp/ruby_syntax.rb b/lib/gitlab/untrusted_regexp/ruby_syntax.rb index 1f1da592ce0..db01875d080 100644 --- a/lib/gitlab/untrusted_regexp/ruby_syntax.rb +++ b/lib/gitlab/untrusted_regexp/ruby_syntax.rb @@ -6,7 +6,7 @@ module Gitlab # and converts that to RE2 representation: # /<regexp>/<flags> class RubySyntax - PATTERN = %r{^/(?<regexp>.*)/(?<flags>[ismU]*)$}.freeze + PATTERN = %r{^/(?<regexp>.*)/(?<flags>[ismU]*)$} # Checks if pattern matches a regexp pattern # but does not enforce it's validity diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 31256101bd2..57d3b3ec6f9 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -260,7 +260,7 @@ module Gitlab return true if /\n|\r/.match?(url) # Google Cloud Storage uses a multi-line, encoded Signature query string - return false if %w(http https).include?(parsed_url.scheme&.downcase) + return false if %w[http https].include?(parsed_url.scheme&.downcase) CGI.unescape(url) =~ /\n|\r/ end diff --git a/lib/gitlab/url_sanitizer.rb b/lib/gitlab/url_sanitizer.rb index 79e124a58f5..20cbde0e700 100644 --- a/lib/gitlab/url_sanitizer.rb +++ b/lib/gitlab/url_sanitizer.rb @@ -26,6 +26,12 @@ module Gitlab #{URI::REGEXP::PATTERN::HOSTPORT} ) }x + # This expression is derived from `URI::REGEXP::PATTERN::USERINFO` but with the + # addition of `{` and `}` in the list of allowed characters to account for the + # possibility of the userinfo portion of a URL containing masked segments. + # e.g. + # http://myuser:{masked_password}@{masked_domain}.com/{masked_hook} + MASKED_USERINFO_REGEX = %r{(?:[\\-_.!~*'()a-zA-Z\d;:&=+$,{}]|%[a-fA-F\d]{2})*} def self.sanitize(content) content.gsub(URI_REGEXP) do |url| @@ -50,6 +56,14 @@ module Gitlab valid?(url, allowed_schemes: ALLOWED_WEB_SCHEMES) end + # The url associated with records like `WebHookLog` may contain masked + # portions represented by paired curly brackets in the URL. As this + # prohibits straightforward parsing of the URL, we can use a variation of + # the existing USERINFO regex for these cases. + def self.sanitize_masked_url(url) + url.gsub(%r{//#{MASKED_USERINFO_REGEX}@}o, '//*****:*****@') + end + def initialize(url, credentials: nil) %i[user password].each do |symbol| credentials[symbol] = credentials[symbol].presence if credentials&.key?(symbol) diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index bd42586731e..450575b7223 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -4,7 +4,6 @@ module Gitlab module Usage class MetricDefinition METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') - SKIP_VALIDATION_STATUS = 'removed' AVAILABLE_STATUSES = %w[active broken].to_set.freeze VALID_SERVICE_PING_STATUSES = %w[active broken].to_set.freeze @@ -26,6 +25,18 @@ module Gitlab events_from_new_structure || events_from_old_structure || {} end + def to_context + return unless %w[redis redis_hll].include?(data_source) + + event_name = if data_source == 'redis_hll' + options[:events].first + elsif data_source == 'redis' + Gitlab::Usage::Metrics::Instrumentations::RedisMetric.new(attributes).redis_key + end + + Gitlab::Tracking::ServicePingContext.new(data_source: data_source, event: event_name) + end + def to_h attributes end @@ -98,6 +109,10 @@ module Gitlab all.select { |definition| definition.attributes[:instrumentation_class].present? && definition.available? } end + def context_for(key_path) + definitions[key_path].to_context + end + def schemer @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH)) end @@ -161,7 +176,7 @@ module Gitlab end def skip_validation? - !!attributes[:skip_validation] || @skip_validation || attributes[:status] == SKIP_VALIDATION_STATUS + !!attributes[:skip_validation] || @skip_validation end def events_from_new_structure diff --git a/lib/gitlab/usage/metrics/instrumentations/batched_background_migration_failed_jobs_metric.rb b/lib/gitlab/usage/metrics/instrumentations/batched_background_migration_failed_jobs_metric.rb index f5d963cf522..807e0ee071a 100644 --- a/lib/gitlab/usage/metrics/instrumentations/batched_background_migration_failed_jobs_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/batched_background_migration_failed_jobs_metric.rb @@ -27,6 +27,10 @@ module Gitlab } end end + + def to_sql + relation.unscope(:order).to_sql + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_connected_agents_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_connected_agents_metric.rb new file mode 100644 index 00000000000..dea3f73c008 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_connected_agents_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountConnectedAgentsMetric < DatabaseMetric + operation :count + + relation do + Clusters::AgentToken.connected + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_total_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_total_metric.rb index 109d2245635..4430c7453e6 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_total_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_total_metric.rb @@ -10,8 +10,8 @@ module Gitlab relation { Project } operation :count - IMPORT_TYPES = %w(gitlab_project gitlab github bitbucket bitbucket_server gitea git manifest - gitlab_migration).freeze + IMPORT_TYPES = %w[gitlab_project gitlab github bitbucket bitbucket_server gitea git manifest + gitlab_migration].freeze def value count(project_relation) + count(entity_relation) diff --git a/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric.rb index e6093691f48..745d147ac88 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_issues_created_manually_from_alerts_metric.rb @@ -13,7 +13,7 @@ module Gitlab cache_start_and_finish_as :issue relation do - Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot) + Issue.with_alert_management_alerts.not_authored_by(::Users::Internal.alert_bot) end def value diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index 2af7c208fce..7133c815e10 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -18,7 +18,7 @@ module Gitlab UnimplementedOperationError = Class.new(StandardError) # rubocop:disable UsageData/InstrumentationSuperclass class << self - IMPLEMENTED_OPERATIONS = %i(count distinct_count estimate_batch_distinct_count sum average).freeze + IMPLEMENTED_OPERATIONS = %i[count distinct_count estimate_batch_distinct_count sum average].freeze private_constant :IMPLEMENTED_OPERATIONS diff --git a/lib/gitlab/usage/metrics/instrumentations/issues_created_from_alerts_metric.rb b/lib/gitlab/usage/metrics/instrumentations/issues_created_from_alerts_metric.rb index e430bc8eb71..d88128216d6 100644 --- a/lib/gitlab/usage/metrics/instrumentations/issues_created_from_alerts_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/issues_created_from_alerts_metric.rb @@ -7,8 +7,7 @@ module Gitlab class IssuesCreatedFromAlertsMetric < NumbersMetric ISSUES_FROM_ALERTS_METRICS = [ IssuesWithAlertManagementAlertsMetric, - IssuesWithPrometheusAlertEvents, - IssuesWithSelfManagedPrometheusAlertEvents + IssuesWithPrometheusAlertEvents ].freeze operation :add diff --git a/lib/gitlab/usage/metrics/instrumentations/issues_with_self_managed_prometheus_alert_events.rb b/lib/gitlab/usage/metrics/instrumentations/issues_with_self_managed_prometheus_alert_events.rb deleted file mode 100644 index fdbaa65bc68..00000000000 --- a/lib/gitlab/usage/metrics/instrumentations/issues_with_self_managed_prometheus_alert_events.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class IssuesWithSelfManagedPrometheusAlertEvents < DatabaseMetric - # this metric is used in IssuesCreatedFromAlertsMetric - # do not report metric directly in service ping - available? { false } - - operation :count - - start { Issue.minimum(:id) } - finish { Issue.maximum(:id) } - - relation { Issue.with_self_managed_prometheus_alert_events } - - cache_start_and_finish_as :issue - end - end - end - end -end diff --git a/lib/gitlab/usage/metrics/instrumentations/numbers_metric.rb b/lib/gitlab/usage/metrics/instrumentations/numbers_metric.rb index 67fcd226a0a..f168837bd50 100644 --- a/lib/gitlab/usage/metrics/instrumentations/numbers_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/numbers_metric.rb @@ -21,7 +21,7 @@ module Gitlab UnimplementedOperationError = Class.new(StandardError) # rubocop:disable UsageData/InstrumentationSuperclass class << self - IMPLEMENTED_OPERATIONS = %i(add).freeze + IMPLEMENTED_OPERATIONS = %i[add].freeze private_constant :IMPLEMENTED_OPERATIONS diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb index ca5e5b706c4..217fa6aca49 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb @@ -48,8 +48,6 @@ module Gitlab end end - private - def redis_key key = metric_event.dup key.prepend("#{prefix}_") if prefix diff --git a/lib/gitlab/usage/time_series_storable.rb b/lib/gitlab/usage/time_series_storable.rb new file mode 100644 index 00000000000..83f62b8c1c7 --- /dev/null +++ b/lib/gitlab/usage/time_series_storable.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module TimeSeriesStorable + # requires a #redis_key(event, date) method to be defined + def keys_for_aggregation(events:, start_date:, end_date:) + # we always keep 1 week of margin + # .end_of_week is necessary to make sure this works for 1 week long periods too + end_date = end_date.end_of_week - 1.week + (start_date.to_date..end_date.to_date).flat_map do |date| + events.map { |event| redis_key(event, date) } + end.uniq + end + + def apply_time_aggregation(key, time) + year_week = time.strftime('%G-%V') + "#{key}-#{year_week}" + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index ab041a31bde..c3378856633 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -20,7 +20,7 @@ module Gitlab class UsageData MAX_GENERATION_TIME_FOR_SAAS = 40.hours - CE_MEMOIZED_VALUES = %i( + CE_MEMOIZED_VALUES = %i[ issue_minimum_id issue_maximum_id project_minimum_id @@ -31,7 +31,7 @@ module Gitlab deployment_maximum_id auth_providers recorded_at - ).freeze + ].freeze class << self include Gitlab::Utils::UsageData @@ -96,7 +96,7 @@ module Gitlab issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data, issues_created_from_alerts: total_alert_issues, incident_issues: count(::Issue.with_issue_type(:incident), start: minimum_id(Issue), finish: maximum_id(Issue)), - alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)), + alert_bot_incident_issues: count(::Issue.authored(::Users::Internal.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)), keys: count(Key), label_lists: count(List.label), lfs_objects: count(LfsObject), @@ -529,7 +529,7 @@ module Gitlab service_desk_issues: count( ::Issue.where( project: projects_with_service_desk, - author: ::User.support_bot, + author: ::Users::Internal.support_bot, confidential: true ) ) diff --git a/lib/gitlab/usage_data/topology.rb b/lib/gitlab/usage_data/topology.rb index b823d6cc2bf..d9f23d2d887 100644 --- a/lib/gitlab/usage_data/topology.rb +++ b/lib/gitlab/usage_data/topology.rb @@ -130,7 +130,7 @@ module Gitlab client.query(query) end - map_instance_labels(node_uname_info, %w(machine sysname release)) + map_instance_labels(node_uname_info, %w[machine sysname release]) end def topology_all_service_memory(client) diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index 21cc9368f44..eb141a2e2f6 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -38,7 +38,7 @@ module Gitlab::UsageDataCounters expanded_template_name = expand_template_name(template_name) results = [expanded_template_name].tap do |result| template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name.chomp('.gitlab-ci.yml')) - data = Gitlab::Ci::Config::Yaml.load!(template.content) + data = Gitlab::Ci::Config::Yaml::Loader.new(template.content).load.content [data[:include]].compact.flatten.each do |ci_include| if ci_include_template = ci_include[:template] result.concat(all_included_templates(ci_include_template)) diff --git a/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb index b30c4b675f9..be566617bd4 100644 --- a/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb @@ -6,7 +6,7 @@ module Gitlab GITLAB_CLI_API_REQUEST_ACTION = 'i_code_review_user_gitlab_cli_api_request' # This regex will match to user agents ending with GitLab CLI or starting with glab/v" - GITLAB_CLI_USER_AGENT_REGEX = %r{(GitLab\sCLI$|^glab/v)}.freeze + GITLAB_CLI_USER_AGENT_REGEX = %r{(GitLab\sCLI$|^glab/v)} class << self def track_api_request_when_trackable(user_agent:, user:) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 53594a27867..f9dc8bd8a3c 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -18,6 +18,7 @@ module Gitlab class << self include Gitlab::Utils::UsageData include Gitlab::Usage::TimeFrame + include Gitlab::Usage::TimeSeriesStorable # Track unique events # @@ -78,13 +79,6 @@ module Gitlab redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end - def keys_for_aggregation(events:, start_date:, end_date:) - end_date = end_date.end_of_week - 1.week - (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date) } - end.flatten.uniq - end - def load_events events = Gitlab::Usage::MetricDefinition.all.map do |d| next unless d.available? @@ -109,7 +103,6 @@ module Gitlab known_events.select { |event| event_names.include?(event[:name]) } end - # Compose the key in order to store events daily or weekly def redis_key(event, time) raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s) diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 54464b63fce..39d9560c0fb 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -38,170 +38,120 @@ module Gitlab class << self def track_issue_created_action(author:, namespace:) - track_internal_action(ISSUE_CREATED, author, namespace) + track_internal_event(ISSUE_CREATED, author, namespace) end def track_issue_title_changed_action(author:, project:) - track_snowplow_action(ISSUE_TITLE_CHANGED, author, project) - track_unique_action(ISSUE_TITLE_CHANGED, author) + track_internal_event(ISSUE_TITLE_CHANGED, author, project) end def track_issue_description_changed_action(author:, project:) - track_snowplow_action(ISSUE_DESCRIPTION_CHANGED, author, project) - track_unique_action(ISSUE_DESCRIPTION_CHANGED, author) + track_internal_event(ISSUE_DESCRIPTION_CHANGED, author, project) end def track_issue_assignee_changed_action(author:, project:) - track_snowplow_action(ISSUE_ASSIGNEE_CHANGED, author, project) - track_unique_action(ISSUE_ASSIGNEE_CHANGED, author) + track_internal_event(ISSUE_ASSIGNEE_CHANGED, author, project) end def track_issue_made_confidential_action(author:, project:) - track_snowplow_action(ISSUE_MADE_CONFIDENTIAL, author, project) - track_unique_action(ISSUE_MADE_CONFIDENTIAL, author) + track_internal_event(ISSUE_MADE_CONFIDENTIAL, author, project) end def track_issue_made_visible_action(author:, project:) - track_snowplow_action(ISSUE_MADE_VISIBLE, author, project) - track_unique_action(ISSUE_MADE_VISIBLE, author) + track_internal_event(ISSUE_MADE_VISIBLE, author, project) end def track_issue_closed_action(author:, project:) - track_snowplow_action(ISSUE_CLOSED, author, project) - track_unique_action(ISSUE_CLOSED, author) + track_internal_event(ISSUE_CLOSED, author, project) end def track_issue_reopened_action(author:, project:) - track_snowplow_action(ISSUE_REOPENED, author, project) - track_unique_action(ISSUE_REOPENED, author) + track_internal_event(ISSUE_REOPENED, author, project) end def track_issue_label_changed_action(author:, project:) - track_snowplow_action(ISSUE_LABEL_CHANGED, author, project) - track_unique_action(ISSUE_LABEL_CHANGED, author) + track_internal_event(ISSUE_LABEL_CHANGED, author, project) end def track_issue_milestone_changed_action(author:, project:) - track_snowplow_action(ISSUE_MILESTONE_CHANGED, author, project) - track_unique_action(ISSUE_MILESTONE_CHANGED, author) + track_internal_event(ISSUE_MILESTONE_CHANGED, author, project) end def track_issue_cross_referenced_action(author:, project:) - track_snowplow_action(ISSUE_CROSS_REFERENCED, author, project) - track_unique_action(ISSUE_CROSS_REFERENCED, author) + track_internal_event(ISSUE_CROSS_REFERENCED, author, project) end def track_issue_moved_action(author:, project:) - track_snowplow_action(ISSUE_MOVED, author, project) - track_unique_action(ISSUE_MOVED, author) + track_internal_event(ISSUE_MOVED, author, project) end def track_issue_related_action(author:, project:) - track_snowplow_action(ISSUE_RELATED, author, project) - track_unique_action(ISSUE_RELATED, author) + track_internal_event(ISSUE_RELATED, author, project) end def track_issue_unrelated_action(author:, project:) - track_snowplow_action(ISSUE_UNRELATED, author, project) - track_unique_action(ISSUE_UNRELATED, author) + track_internal_event(ISSUE_UNRELATED, author, project) end def track_issue_marked_as_duplicate_action(author:, project:) - track_snowplow_action(ISSUE_MARKED_AS_DUPLICATE, author, project) - track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author) + track_internal_event(ISSUE_MARKED_AS_DUPLICATE, author, project) end def track_issue_locked_action(author:, project:) - track_snowplow_action(ISSUE_LOCKED, author, project) - track_unique_action(ISSUE_LOCKED, author) + track_internal_event(ISSUE_LOCKED, author, project) end def track_issue_unlocked_action(author:, project:) - track_snowplow_action(ISSUE_UNLOCKED, author, project) - track_unique_action(ISSUE_UNLOCKED, author) + track_internal_event(ISSUE_UNLOCKED, author, project) end def track_issue_designs_added_action(author:, project:) - track_snowplow_action(ISSUE_DESIGNS_ADDED, author, project) - track_unique_action(ISSUE_DESIGNS_ADDED, author) + track_internal_event(ISSUE_DESIGNS_ADDED, author, project) end def track_issue_designs_modified_action(author:, project:) - track_snowplow_action(ISSUE_DESIGNS_MODIFIED, author, project) - track_unique_action(ISSUE_DESIGNS_MODIFIED, author) + track_internal_event(ISSUE_DESIGNS_MODIFIED, author, project) end def track_issue_designs_removed_action(author:, project:) - track_snowplow_action(ISSUE_DESIGNS_REMOVED, author, project) - track_unique_action(ISSUE_DESIGNS_REMOVED, author) + track_internal_event(ISSUE_DESIGNS_REMOVED, author, project) end def track_issue_due_date_changed_action(author:, project:) - track_snowplow_action(ISSUE_DUE_DATE_CHANGED, author, project) - track_unique_action(ISSUE_DUE_DATE_CHANGED, author) + track_internal_event(ISSUE_DUE_DATE_CHANGED, author, project) end def track_issue_time_estimate_changed_action(author:, project:) - track_snowplow_action(ISSUE_TIME_ESTIMATE_CHANGED, author, project) - track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author) + track_internal_event(ISSUE_TIME_ESTIMATE_CHANGED, author, project) end def track_issue_time_spent_changed_action(author:, project:) - track_snowplow_action(ISSUE_TIME_SPENT_CHANGED, author, project) - track_unique_action(ISSUE_TIME_SPENT_CHANGED, author) + track_internal_event(ISSUE_TIME_SPENT_CHANGED, author, project) end def track_issue_comment_added_action(author:, project:) - track_snowplow_action(ISSUE_COMMENT_ADDED, author, project) - track_unique_action(ISSUE_COMMENT_ADDED, author) + track_internal_event(ISSUE_COMMENT_ADDED, author, project) end def track_issue_comment_edited_action(author:, project:) - track_snowplow_action(ISSUE_COMMENT_EDITED, author, project) - track_unique_action(ISSUE_COMMENT_EDITED, author) + track_internal_event(ISSUE_COMMENT_EDITED, author, project) end def track_issue_comment_removed_action(author:, project:) - track_snowplow_action(ISSUE_COMMENT_REMOVED, author, project) - track_unique_action(ISSUE_COMMENT_REMOVED, author) + track_internal_event(ISSUE_COMMENT_REMOVED, author, project) end def track_issue_cloned_action(author:, project:) - track_snowplow_action(ISSUE_CLONED, author, project) - track_unique_action(ISSUE_CLONED, author) + track_internal_event(ISSUE_CLONED, author, project) end def track_issue_design_comment_removed_action(author:, project:) - track_snowplow_action(ISSUE_DESIGN_COMMENT_REMOVED, author, project) - track_unique_action(ISSUE_DESIGN_COMMENT_REMOVED, author) + track_internal_event(ISSUE_DESIGN_COMMENT_REMOVED, author, project) end private - def track_snowplow_action(event_name, author, container) - namespace, project = get_params_from_container(container) - - return unless author - - Gitlab::Tracking.event( - ISSUE_CATEGORY, - ISSUE_ACTION, - label: ISSUE_LABEL, - property: event_name, - project: project, - namespace: namespace, - user: author, - context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context] - ) - end - - def track_unique_action(event_name, author) - return unless author - - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: author.id) - end - - def track_internal_action(event_name, author, container) + def track_internal_event(event_name, author, container) return unless author namespace, project = get_params_from_container(container) diff --git a/lib/gitlab/usage_data_counters/jetbrains_plugin_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/jetbrains_plugin_activity_unique_counter.rb index f3d5be5e28f..42387b0d7f4 100644 --- a/lib/gitlab/usage_data_counters/jetbrains_plugin_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/jetbrains_plugin_activity_unique_counter.rb @@ -4,7 +4,7 @@ module Gitlab module UsageDataCounters module JetBrainsPluginActivityUniqueCounter JETBRAINS_API_REQUEST_ACTION = 'i_code_review_user_jetbrains_api_request' - JETBRAINS_USER_AGENT_REGEX = /\Agitlab-jetbrains-plugin/.freeze + JETBRAINS_USER_AGENT_REGEX = /\Agitlab-jetbrains-plugin/ class << self def track_api_request_when_trackable(user_agent:, user:) diff --git a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb index 9e8c207a19a..f4a1dd692b7 100644 --- a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb +++ b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb @@ -10,6 +10,7 @@ module Gitlab flux_git_push_notifications_total k8s_api_proxy_requests_via_ci_access k8s_api_proxy_requests_via_user_access + k8s_api_proxy_requests_via_pat_access ].freeze class << self diff --git a/lib/gitlab/usage_data_counters/vscode_extension_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/vscode_extension_activity_unique_counter.rb index 703c4885b04..7b3df134ba3 100644 --- a/lib/gitlab/usage_data_counters/vscode_extension_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/vscode_extension_activity_unique_counter.rb @@ -4,7 +4,7 @@ module Gitlab module UsageDataCounters module VSCodeExtensionActivityUniqueCounter VS_CODE_API_REQUEST_ACTION = 'i_code_review_user_vs_code_api_request' - VS_CODE_USER_AGENT_REGEX = /\Avs-code-gitlab-workflow/.freeze + VS_CODE_USER_AGENT_REGEX = /\Avs-code-gitlab-workflow/ class << self def track_api_request_when_trackable(user_agent:, user:) diff --git a/lib/gitlab/utils/link_header_parser.rb b/lib/gitlab/utils/link_header_parser.rb index d98c237baf3..a766be93b7e 100644 --- a/lib/gitlab/utils/link_header_parser.rb +++ b/lib/gitlab/utils/link_header_parser.rb @@ -19,9 +19,9 @@ module Gitlab # } # } class LinkHeaderParser - REL_PATTERN = %r{rel="(\w+)"}.freeze + REL_PATTERN = %r{rel="(\w+)"} # to avoid parse really long URIs we limit the amount of characters allowed - URI_PATTERN = %r{<(.{1,500})>}.freeze + URI_PATTERN = %r{<(.{1,500})>} def initialize(header) @header = header diff --git a/lib/gitlab/utils/markdown.rb b/lib/gitlab/utils/markdown.rb index c041953e7c8..750789430d9 100644 --- a/lib/gitlab/utils/markdown.rb +++ b/lib/gitlab/utils/markdown.rb @@ -3,8 +3,8 @@ module Gitlab module Utils module Markdown - PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze - PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate|free|bronze|silver|gold)(\s+(all|only|self|saas))?\)\**/.freeze + PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u + PRODUCT_SUFFIX = /\s*\**\((premium|ultimate|free|beta|experiment)(\s+(all|self|saas))?(\s+(beta|experiment))?\)\**/ def string_to_anchor(string) string diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 10370811bb5..9bc37746349 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -3,6 +3,9 @@ require 'gitlab/utils/all' require_relative '../environment' +# See https://docs.gitlab.com/ee/development/utilities.html#override for +# more information + module Gitlab module Utils module Override diff --git a/lib/gitlab/utils/sanitize_node_link.rb b/lib/gitlab/utils/sanitize_node_link.rb index 183741ff5f5..2723ef137c3 100644 --- a/lib/gitlab/utils/sanitize_node_link.rb +++ b/lib/gitlab/utils/sanitize_node_link.rb @@ -5,8 +5,8 @@ require_dependency 'gitlab/utils' module Gitlab module Utils module SanitizeNodeLink - UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze - ATTRS_TO_SANITIZE = %w(href src data-src data-canonical-src).freeze + UNSAFE_PROTOCOLS = %w[data javascript vbscript].freeze + ATTRS_TO_SANITIZE = %w[href src data-src data-canonical-src].freeze # sanitize 6.0 requires only a context argument. Do not add any default # arguments to this method. diff --git a/lib/gitlab/uuid.rb b/lib/gitlab/uuid.rb index a3abe90a412..9087e20bbb8 100644 --- a/lib/gitlab/uuid.rb +++ b/lib/gitlab/uuid.rb @@ -9,7 +9,7 @@ module Gitlab production: "58dc0f06-936c-43b3-93bb-71693f1b6570" }.freeze - UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{12}/.freeze + UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{12}/ class << self def v5(name, namespace_id: default_namespace_id) diff --git a/lib/gitlab/web_hooks/rate_limiter.rb b/lib/gitlab/web_hooks/rate_limiter.rb index 73d59f6f786..e68e97a29c5 100644 --- a/lib/gitlab/web_hooks/rate_limiter.rb +++ b/lib/gitlab/web_hooks/rate_limiter.rb @@ -9,7 +9,7 @@ module Gitlab NO_LIMIT = 0 # SystemHooks (instance admin hooks) and ServiceHooks (integration hooks) # are not rate-limited. - EXCLUDED_HOOK_TYPES = %w(SystemHook ServiceHook).freeze + EXCLUDED_HOOK_TYPES = %w[SystemHook ServiceHook].freeze def initialize(hook) @hook = hook diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 79131404465..d19dd6cd856 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -15,7 +15,7 @@ module Gitlab NOTIFICATION_PREFIX = 'workhorse:notifications:' ALLOWED_GIT_HTTP_ACTIONS = %w[git_receive_pack git_upload_pack info_refs].freeze DETECT_HEADER = 'Gitlab-Workhorse-Detect-Content-Type' - ARCHIVE_FORMATS = %w(zip tar.gz tar.bz2 tar).freeze + ARCHIVE_FORMATS = %w[zip tar.gz tar.bz2 tar].freeze include JwtAuthenticatable @@ -184,11 +184,17 @@ module Gitlab ] end - def send_dependency(headers, url) + def send_dependency(headers, url, upload_config: {}) params = { - 'Header' => headers, - 'Url' => url + 'Headers' => headers.transform_values { |v| Array.wrap(v) }, + 'Url' => url, + 'UploadConfig' => { + 'Method' => upload_config[:method], + 'Url' => upload_config[:url], + 'Headers' => (upload_config[:headers] || {}).transform_values { |v| Array.wrap(v) } + }.compact_blank! } + params.compact_blank! [ SEND_DATA_HEADER, @@ -227,8 +233,12 @@ module Gitlab Gitlab.config.workhorse.secret_file end + def cleanup_key(key) + with_redis { |redis| redis.del(key) } + end + def set_key_and_notify(key, value, expire: nil, overwrite: true) - Gitlab::Redis::SharedState.with do |redis| + with_redis do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result redis.publish(NOTIFICATION_PREFIX + key, value) @@ -249,6 +259,15 @@ module Gitlab protected + def with_redis(&blk) + if Feature.enabled?(:use_primary_and_secondary_stores_for_workhorse) || + Feature.enabled?(:use_primary_store_as_default_for_workhorse) + Gitlab::Redis::Workhorse.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + else + Gitlab::Redis::SharedState.with(&blk) # rubocop:disable CodeReuse/ActiveRecord + end + end + # This is the outermost encoding of a senddata: header. It is safe for # inclusion in HTTP response headers def encode(hash) diff --git a/lib/gitlab/x509/certificate.rb b/lib/gitlab/x509/certificate.rb index 98688f504eb..e1ba5cdac7d 100644 --- a/lib/gitlab/x509/certificate.rb +++ b/lib/gitlab/x509/certificate.rb @@ -3,7 +3,7 @@ module Gitlab module X509 class Certificate - CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze + CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/ attr_reader :key, :cert, :ca_certs diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index 649e5379927..74d861c8701 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -39,6 +39,8 @@ module Gitlab return :unverified if x509_certificate.nil? || x509_certificate.revoked? || + certificate_subject.nil? || + certificate_crl.nil? || !verified_signature || signed_by_user.nil? @@ -127,6 +129,7 @@ module Gitlab def certificate_crl extension = get_certificate_extension('crlDistributionPoints') + return if extension.nil? crl_url = nil @@ -185,7 +188,7 @@ module Gitlab end def x509_issuer - return if verified_signature.nil? || issuer_subject_key_identifier.nil? || certificate_crl.nil? + return if verified_signature.nil? || issuer_subject_key_identifier.nil? || certificate_issuer.nil? attributes = { subject_key_identifier: issuer_subject_key_identifier, diff --git a/lib/gitlab/zoom_link_extractor.rb b/lib/gitlab/zoom_link_extractor.rb index 7ac14eb2d4f..92b3d6d5d3c 100644 --- a/lib/gitlab/zoom_link_extractor.rb +++ b/lib/gitlab/zoom_link_extractor.rb @@ -8,7 +8,7 @@ module Gitlab class ZoomLinkExtractor - ZOOM_REGEXP = %r{https://(?:[\w-]+\.)?zoom\.us/(?:s|j|my)/\S+}.freeze + ZOOM_REGEXP = %r{https://(?:[\w-]+\.)?zoom\.us/(?:s|j|my)/\S+} def initialize(text) @text = text.to_s |