diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-06-20 13:43:29 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-06-20 13:43:29 +0300 |
commit | 3b1af5cc7ed2666ff18b718ce5d30fa5a2756674 (patch) | |
tree | 3bc4a40e0ee51ec27eabf917c537033c0c5b14d4 /lib/gitlab | |
parent | 9bba14be3f2c211bf79e15769cd9b77bc73a13bc (diff) |
Add latest changes from gitlab-org/gitlab@16-1-stable-eev16.1.0-rc42
Diffstat (limited to 'lib/gitlab')
292 files changed, 3772 insertions, 3300 deletions
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index bafda11170a..f1777e059ed 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -23,6 +23,7 @@ module Gitlab PROTECTION_DEV_CAN_PUSH = 1 PROTECTION_FULL = 2 PROTECTION_DEV_CAN_MERGE = 3 + PROTECTION_DEV_CAN_INITIAL_PUSH = 4 # Default project creation level NO_ONE_PROJECT_ACCESS = 0 @@ -95,6 +96,11 @@ module Gitlab label: s_('DefaultBranchProtection|Fully protected'), help_text: s_('DefaultBranchProtection|Developers cannot push new commits, but maintainers can. No one can force push.'), value: PROTECTION_FULL + }, + { + label: s_('DefaultBranchProtection|Fully protected after initial push'), + help_text: s_('DefaultBranchProtection|Developers can push the initial commit to a repository, but none afterward. Maintainers can always push. No one can force push.'), + value: PROTECTION_DEV_CAN_INITIAL_PUSH } ] end diff --git a/lib/gitlab/access/branch_protection.rb b/lib/gitlab/access/branch_protection.rb index 339a99eb068..6ac8de407b0 100644 --- a/lib/gitlab/access/branch_protection.rb +++ b/lib/gitlab/access/branch_protection.rb @@ -34,6 +34,10 @@ module Gitlab level == PROTECTION_DEV_CAN_PUSH end + def developer_can_initial_push? + level == PROTECTION_DEV_CAN_INITIAL_PUSH + end + def developer_can_merge? level == PROTECTION_DEV_CAN_MERGE end diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index 01dcb95eab5..5b136431ce7 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -181,7 +181,6 @@ module Gitlab end end - # Overriden in EE::Gitlab::AlertManagement::Payload::Generic def value_for_paths(paths) target_path = paths.find { |path| payload&.dig(*path) } diff --git a/lib/gitlab/alert_management/payload/prometheus.rb b/lib/gitlab/alert_management/payload/prometheus.rb index 4c36ebbf3aa..76f3da8366b 100644 --- a/lib/gitlab/alert_management/payload/prometheus.rb +++ b/lib/gitlab/alert_management/payload/prometheus.rb @@ -94,6 +94,10 @@ module Gitlab project && title && starts_at_raw end + def source + integration&.name || monitoring_tool + end + private override :severity_mapping @@ -131,3 +135,5 @@ module Gitlab end end end + +Gitlab::AlertManagement::Payload::Prometheus.prepend_mod diff --git a/lib/gitlab/analytics/date_filler.rb b/lib/gitlab/analytics/date_filler.rb index aa3db9f3635..33ebe269f26 100644 --- a/lib/gitlab/analytics/date_filler.rb +++ b/lib/gitlab/analytics/date_filler.rb @@ -32,7 +32,7 @@ module Gitlab # End date of the range # # **period** - # Specifies the period in wich the dates should be generated. Options: + # Specifies the period in which the dates should be generated. Options: # # - :day, generate date-value pair for each day in the given period # - :week, generate date-value pair for each week (beginning of the week date) diff --git a/lib/gitlab/api_authentication/token_locator.rb b/lib/gitlab/api_authentication/token_locator.rb index df342905d2e..5656ea0d120 100644 --- a/lib/gitlab/api_authentication/token_locator.rb +++ b/lib/gitlab/api_authentication/token_locator.rb @@ -8,22 +8,23 @@ module Gitlab include ActiveModel::Validations include ActionController::HttpAuthentication::Basic + VALID_LOCATIONS = %i[ + http_basic_auth + http_token + http_bearer_token + http_deploy_token_header + http_job_token_header + http_private_token_header + http_header + token_param + ].freeze + attr_reader :location - validates :location, inclusion: { - in: %i[ - http_basic_auth - http_token - http_bearer_token - http_deploy_token_header - http_job_token_header - http_private_token_header - token_param - ] - } + validates :location, inclusion: { in: VALID_LOCATIONS } def initialize(location) - @location = location + @location = extract_location(location) validate! end @@ -41,6 +42,8 @@ module Gitlab extract_from_http_job_token_header request when :http_private_token_header extract_from_http_private_token_header request + when :http_header + extract_from_http_header request when :token_param extract_from_token_param request end @@ -48,6 +51,16 @@ module Gitlab private + def extract_location(location) + case location + when Symbol + location + when Hash + result, @token_identifier = location.detect { |k, _v| VALID_LOCATIONS.include?(k) } + result + end + end + def extract_from_http_basic_auth(request) username, password = user_name_and_password(request) return unless username.present? && password.present? @@ -96,6 +109,13 @@ module Gitlab UsernameAndPassword.new(nil, password) end + + def extract_from_http_header(request) + password = request.headers[@token_identifier] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end end end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index a8e74cbd7e6..8d7712951e1 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -45,7 +45,7 @@ module Gitlab auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute }, search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, - gitlab_shell_operation: { threshold: 600, interval: 1.minute }, + gitlab_shell_operation: { threshold: application_settings.gitlab_shell_operation_limit, interval: 1.minute }, pipelines_create: { threshold: -> { application_settings.pipeline_limit_per_project_user_sha }, interval: 1.minute }, temporary_email_failure: { threshold: 300, interval: 1.day }, permanent_email_failure: { threshold: 5, interval: 1.day }, diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index d55f2bc8ac9..955cb14594f 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -70,7 +70,8 @@ module Gitlab .merge({ # Define the Kroki server URL from the settings. # This attribute cannot be overridden from the AsciiDoc document. - 'kroki-server-url' => Gitlab::CurrentSettings.kroki_url + 'kroki-server-url' => Gitlab::CurrentSettings.kroki_url, + 'allow-uri-read' => Gitlab::CurrentSettings.wiki_asciidoc_allow_uri_includes }), extensions: extensions } diff --git a/lib/gitlab/asciidoc/include_processor.rb b/lib/gitlab/asciidoc/include_processor.rb index 6c4ecc04cdc..ae83dbedf04 100644 --- a/lib/gitlab/asciidoc/include_processor.rb +++ b/lib/gitlab/asciidoc/include_processor.rb @@ -9,6 +9,8 @@ module Gitlab class IncludeProcessor < Asciidoctor::IncludeExt::IncludeProcessor extend ::Gitlab::Utils::Override + NoData = Class.new(StandardError) + def initialize(context) super(logger: Gitlab::AppLogger) @@ -16,6 +18,7 @@ module Gitlab @repository = context[:repository] || context[:project].try(:repository) @max_includes = context[:max_includes].to_i @included = [] + @included_content = {} # Note: Asciidoctor calls #freeze on extensions, so we can't set new # instance variables after initialization. @@ -31,9 +34,10 @@ module Gitlab doc = reader.document max_include_depth = doc.attributes.fetch('max-include-depth').to_i + allow_uri_read = doc.attributes.fetch('allow-uri-read', false) return false if max_include_depth < 1 - return false if target_http?(target) + return false if target_http?(target) && !allow_uri_read return false if included.size >= max_includes true @@ -42,6 +46,7 @@ module Gitlab override :resolve_target_path def resolve_target_path(target, reader) return unless repository.try(:exists?) + return target if target_http?(target) base_path = reader.include_stack.empty? ? requested_path : reader.file path = resolve_relative_path(target, base_path) @@ -51,12 +56,15 @@ module Gitlab override :read_lines def read_lines(filename, selector) - blob = read_blob(ref, filename) + content = read_content(filename) + raise NoData, filename if content.nil? + + included << filename if selector - blob.data.each_line.select.with_index(1, &selector) + content.each_line.select.with_index(1, &selector) else - blob.data + content.lines end end @@ -67,7 +75,17 @@ module Gitlab private - attr_reader :context, :repository, :cache, :max_includes, :included + attr_reader :context, :repository, :cache, :max_includes, :included, :included_content + + def read_content(filename) + return included_content[filename] if included_content.key?(filename) + + included_content[filename] = if target_http?(filename) + read_uri(filename) + else + read_blob(ref, filename) + end + end # Gets a Blob at a path for a specific revision. # This method will check that the Blob exists and contains readable text. @@ -75,16 +93,22 @@ module Gitlab # revision - The String SHA1. # path - The String file path. # - # Returns a Blob + # Returns a string containing the blob content def read_blob(ref, filename) blob = repository&.blob_at(ref, filename) - raise 'Blob not found' unless blob - raise 'File is not readable' unless blob.readable_text? + raise NoData, 'Blob not found' unless blob + raise NoData, 'File is not readable' unless blob.readable_text? - included << filename + blob.data + end + + def read_uri(uri) + r = Gitlab::HTTP.get(uri) + + raise NoData, uri unless r.success? - blob + r.body end # Resolves the given relative path of file in repository into canonical diff --git a/lib/gitlab/audit/auditor.rb b/lib/gitlab/audit/auditor.rb index e3d2b394404..a59237fbb1f 100644 --- a/lib/gitlab/audit/auditor.rb +++ b/lib/gitlab/audit/auditor.rb @@ -77,12 +77,12 @@ module Gitlab @authentication_provider = @context[:authentication_provider] # TODO: Remove this code once we close https://gitlab.com/gitlab-org/gitlab/-/issues/367870 - return unless @is_audit_event_yaml_defined + return if @is_audit_event_yaml_defined - # rubocop:disable Gitlab/RailsLogger - Rails.logger.warn('WARNING: Logging audit events without an event type definition will be deprecated soon.') - Rails.logger.warn('See https://docs.gitlab.com/ee/development/audit_event_guide/#event-type-definitions') - # rubocop:enable Gitlab/RailsLogger + message = 'Logging audit events without an event type definition will be deprecated soon ' \ + '(https://docs.gitlab.com/ee/development/audit_event_guide/#event-type-definitions)' + + Gitlab::AppLogger.warn(message: message, event_type: @name) end def single_audit diff --git a/lib/gitlab/audit/type/definition.rb b/lib/gitlab/audit/type/definition.rb index 81c88a3a0ae..772023616b8 100644 --- a/lib/gitlab/audit/type/definition.rb +++ b/lib/gitlab/audit/type/definition.rb @@ -13,6 +13,10 @@ module Gitlab validate :validate_schema validate :validate_file_name + def self.declarative_policy_class + 'AuditEvents::DefinitionPolicy' + end + InvalidAuditEventTypeError = Class.new(StandardError) AUDIT_EVENT_TYPE_SCHEMA_PATH = Rails.root.join('config', 'audit_events', 'types', 'type_schema.json') @@ -78,6 +82,12 @@ module Gitlab definitions.keys.map(&:to_s) end + def names_with_category + definitions.map do |event_name, value| + { event_name: event_name, feature_category: value.attributes[:feature_category] } + end + end + def defined?(key) get(key).present? end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 9268fdd8519..83d94d168a0 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -401,7 +401,7 @@ module Gitlab scopes = non_admin_available_scopes if resource.admin? # rubocop: disable Cop/UserAdmin - scopes += Feature.enabled?(:admin_mode_for_api) ? ADMIN_SCOPES : [SUDO_SCOPE] + scopes += ADMIN_SCOPES end scopes diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index d1eede65f0c..cce08750296 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -78,7 +78,7 @@ module Gitlab def get_from_auth_hash_or_info(key) if auth_hash.key?(key) coerce_utf8(auth_hash[key]) - elsif auth_hash.key?(:extra) && auth_hash.extra.key?(:raw_info) && !auth_hash.extra.raw_info[key].nil? + elsif auth_hash.key?(:extra) && auth_hash.extra.key?(:raw_info) && !auth_hash.extra.raw_info[key].blank? coerce_utf8(auth_hash.extra.raw_info[key]) else get_info(key) diff --git a/lib/gitlab/auth/saml/auth_hash.rb b/lib/gitlab/auth/saml/auth_hash.rb index a2b0dfd5c66..592d88264e9 100644 --- a/lib/gitlab/auth/saml/auth_hash.rb +++ b/lib/gitlab/auth/saml/auth_hash.rb @@ -5,7 +5,7 @@ module Gitlab module Saml class AuthHash < Gitlab::Auth::OAuth::AuthHash def groups - Array.wrap(get_raw(Gitlab::Auth::Saml::Config.groups)) + Array.wrap(get_raw(Gitlab::Auth::Saml::Config.new(auth_hash.provider).groups)) end def authn_context diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 815130aeee2..7524d8b9f85 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -8,26 +8,32 @@ module Gitlab def enabled? ::AuthHelper.saml_providers.any? end + end - def options - Gitlab::Auth::OAuth::Provider.config_for('saml') - end + DEFAULT_PROVIDER_NAME = 'saml' - def upstream_two_factor_authn_contexts - options.args[:upstream_two_factor_authn_contexts] - end + def initialize(provider = DEFAULT_PROVIDER_NAME) + @provider = provider + end - def groups - options[:groups_attribute] - end + def options + Gitlab::Auth::OAuth::Provider.config_for(@provider) + end - def external_groups - options[:external_groups] - end + def upstream_two_factor_authn_contexts + options.args[:upstream_two_factor_authn_contexts] + end - def admin_groups - options[:admin_groups] - end + def groups + options[:groups_attribute] + end + + def external_groups + options[:external_groups] + end + + def admin_groups + options[:admin_groups] end end end diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index d14da41deb6..6f72f185c8d 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -43,7 +43,7 @@ module Gitlab protected def saml_config - Gitlab::Auth::Saml::Config + Gitlab::Auth::Saml::Config.new(auth_hash.provider) end def auto_link_saml_user? diff --git a/lib/gitlab/authorized_keys.rb b/lib/gitlab/authorized_keys.rb index e7eba65bea8..3e529a0d2f3 100644 --- a/lib/gitlab/authorized_keys.rb +++ b/lib/gitlab/authorized_keys.rb @@ -149,7 +149,7 @@ module Gitlab raise KeyError, "Invalid public_key: #{key.inspect}" end - %Q(command="#{command(id)}",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty #{strip(key)}) + %(command="#{command(id)}",no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty #{strip(key)}) end def command(id) diff --git a/lib/gitlab/avatar_cache.rb b/lib/gitlab/avatar_cache.rb index ed00a279299..f4dcd6f7910 100644 --- a/lib/gitlab/avatar_cache.rb +++ b/lib/gitlab/avatar_cache.rb @@ -7,7 +7,7 @@ module Gitlab # immediate cache expiry of all avatar caches. # # @return [Integer] - VERSION = 1 + VERSION = 2 # @return [Symbol] BASE_KEY = :avatar_cache @@ -65,10 +65,8 @@ module Gitlab keys = emails.map { |email| email_key(email) } Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - if ::Feature.enabled?(:use_pipeline_over_multikey) - redis.pipelined do |pipeline| - keys.each { |key| pipeline.unlink(key) } - end.sum + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::ClusterUtil.batch_unlink(keys, redis) else redis.unlink(*keys) end diff --git a/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb b/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb deleted file mode 100644 index 63112b52584..00000000000 --- a/lib/gitlab/background_migration/backfill_ci_queuing_tables.rb +++ /dev/null @@ -1,153 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Ensure queuing entries are present even if admins skip upgrades. - class BackfillCiQueuingTables - class Namespace < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'namespaces' - self.inheritance_column = :_type_disabled - end - - class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'projects' - - belongs_to :namespace - has_one :ci_cd_settings, class_name: 'Gitlab::BackgroundMigration::BackfillCiQueuingTables::ProjectCiCdSetting' - - def group_runners_enabled? - return false unless ci_cd_settings - - ci_cd_settings.group_runners_enabled? - end - end - - class ProjectCiCdSetting < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'project_ci_cd_settings' - end - - class Taggings < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'taggings' - end - - module Ci - class Build < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'ci_builds' - self.inheritance_column = :_type_disabled - - belongs_to :project - - scope :pending, -> do - where(status: :pending, type: 'Ci::Build', runner_id: nil) - end - - def self.each_batch(of: 1000, column: :id, order: { runner_id: :asc, id: :asc }, order_hint: nil) - start = except(:select).select(column).reorder(order) - start = start.take - return unless start - - start_id = start[column] - arel_table = self.arel_table - - 1.step do |index| - start_cond = arel_table[column].gteq(start_id) - stop = except(:select).select(column).where(start_cond).reorder(order) - stop = stop.offset(of).limit(1).take - relation = where(start_cond) - - if stop - stop_id = stop[column] - start_id = stop_id - stop_cond = arel_table[column].lt(stop_id) - relation = relation.where(stop_cond) - end - - # Any ORDER BYs are useless for this relation and can lead to less - # efficient UPDATE queries, hence we get rid of it. - relation = relation.except(:order) - - # Using unscoped is necessary to prevent leaking the current scope used by - # ActiveRecord to chain `each_batch` method. - unscoped { yield relation, index } - - break unless stop - end - end - - def tags_ids - BackfillCiQueuingTables::Taggings - .where(taggable_id: id, taggable_type: 'CommitStatus') - .pluck(:tag_id) - end - end - - class PendingBuild < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'ci_pending_builds' - - class << self - def upsert_from_build!(build) - entry = self.new(args_from_build(build)) - - self.upsert( - entry.attributes.compact, - returning: %w[build_id], - unique_by: :build_id) - end - - def args_from_build(build) - project = build.project - - { - build_id: build.id, - project_id: build.project_id, - protected: build.protected?, - namespace_id: project.namespace_id, - tag_ids: build.tags_ids, - instance_runners_enabled: project.shared_runners_enabled?, - namespace_traversal_ids: namespace_traversal_ids(project) - } - end - - def namespace_traversal_ids(project) - if project.group_runners_enabled? - project.namespace.traversal_ids - else - [] - end - end - end - end - end - - BATCH_SIZE = 100 - - def perform(start_id, end_id) - scope = BackfillCiQueuingTables::Ci::Build.pending.where(id: start_id..end_id) - pending_builds_query = BackfillCiQueuingTables::Ci::PendingBuild - .where('ci_builds.id = ci_pending_builds.build_id') - .select(1) - - scope.each_batch(of: BATCH_SIZE) do |builds| - builds = builds.where('NOT EXISTS (?)', pending_builds_query) - builds = builds.includes(:project, project: [:namespace, :ci_cd_settings]) - - builds.each do |build| - BackfillCiQueuingTables::Ci::PendingBuild.upsert_from_build!(build) - end - end - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_code_suggestions_namespace_settings.rb b/lib/gitlab/background_migration/backfill_code_suggestions_namespace_settings.rb new file mode 100644 index 00000000000..2d3bb4bbafa --- /dev/null +++ b/lib/gitlab/background_migration/backfill_code_suggestions_namespace_settings.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class sets default `code_suggestions` values on the namespace_settings table. + # For group namespace, set this to enabled. + # For user namespace, set this to disabled. + class BackfillCodeSuggestionsNamespaceSettings < BatchedMigrationJob + feature_category :code_suggestions + operation_name :update_all + + TYPE_VALUE_PAIRS = [ + { type: 'Group', value: true }, + { type: 'User', value: false } + ].freeze + + NAMESPACES_JOIN = <<~SQL + INNER JOIN namespaces + ON namespaces.id = namespace_settings.namespace_id + SQL + + def perform + TYPE_VALUE_PAIRS.each do |pair| + each_sub_batch do |sub_batch| + sub_batch.joins(NAMESPACES_JOIN) + .where(namespaces: { type: pair[:type] }) + .update_all(code_suggestions: pair[:value]) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_group_features.rb b/lib/gitlab/background_migration/backfill_group_features.rb deleted file mode 100644 index c45dcad5b2d..00000000000 --- a/lib/gitlab/background_migration/backfill_group_features.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill group_features for an array of groups - class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BatchedMigrationJob - job_arguments :batch_size - operation_name :upsert_group_features - feature_category :database - - def perform - each_sub_batch( - batching_arguments: { order_hint: :type }, - batching_scope: ->(relation) { relation.where(type: 'Group') } - ) do |sub_batch| - upsert_group_features(sub_batch) - end - end - - private - - def upsert_group_features(relation) - connection.execute( - <<~SQL - INSERT INTO group_features (group_id, created_at, updated_at) - SELECT namespaces.id as group_id, now(), now() - FROM namespaces - WHERE namespaces.type = 'Group' AND namespaces.id IN(#{relation.select(:id).limit(batch_size).to_sql}) - ON CONFLICT (group_id) DO NOTHING; - SQL - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb deleted file mode 100644 index 0585924cb7b..00000000000 --- a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfills the `routes.namespace_id` column, by copying source_id value - # (for groups and user namespaces source_id == namespace_id) - class BackfillNamespaceIdForNamespaceRoute - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:update_all) do - sub_batch.update_all('namespace_id=source_id') - end - - pause_ms = [0, pause_ms].max - sleep(pause_ms * 0.001) - end - end - - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - - private - - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: ApplicationRecord.connection) - .joins('inner join namespaces on routes.source_id = namespaces.id') - .where(source_key_column => start_id..stop_id) - .where(namespace_id: nil) - .where(source_type: 'Namespace') - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_resource_link_events.rb b/lib/gitlab/background_migration/backfill_resource_link_events.rb new file mode 100644 index 00000000000..a2499e90e1f --- /dev/null +++ b/lib/gitlab/background_migration/backfill_resource_link_events.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills resource_link_events from system_note_metadata and notes records + class BackfillResourceLinkEvents < BatchedMigrationJob + operation_name :backfill_resource_link_events + feature_category :team_planning + + # AR model for resource_link_events inlined + class ResourceLinkEvent < ApplicationRecord + self.table_name = 'resource_link_events' + + enum action: { + add: 1, + remove: 2 + } + end + + scope_to ->(relation) { relation.where("action='relate_to_parent' OR action='unrelate_from_parent'") } + + def perform + each_sub_batch do |sub_batch| + values_subquery = resource_link_event_values_query(sub_batch.select(:id).to_sql) + + connection.execute(<<~SQL) + INSERT INTO resource_link_events (action, issue_id, child_work_item_id, user_id, created_at, system_note_metadata_id) + #{values_subquery} + ON CONFLICT (system_note_metadata_id) DO NOTHING; + SQL + end + end + + def resource_link_event_values_query(ids_subquery) + <<~SQL + SELECT + CASE WHEN system_note_metadata.action='relate_to_parent' THEN #{ResourceLinkEvent.actions[:add]} + ELSE #{ResourceLinkEvent.actions[:remove]} + END AS action, + parent_issues.id AS issue_id, + notes.noteable_id AS child_work_item_id, + notes.author_id AS user_id, + system_note_metadata.created_at AS created_at, + system_note_metadata.id AS system_note_metadata_id + FROM system_note_metadata + INNER JOIN notes ON system_note_metadata.note_id = notes.id + INNER JOIN issues as work_items ON work_items.id = notes.noteable_id, + LATERAL ( + -- This lateral join searches for the id of the parent issue. + -- + -- When a child work item is added to its parent, + -- "relate_to_parent" is recorded as `system_note_metadata.action` + -- and a note records to which parent the child work item is added e.g, "added #1 (iid) as parent". + -- + -- Based on the iid of the parent extracted from the note and using the child work item's project id, + -- we can find out the id of the parent issue. + SELECT issues.id + FROM issues + WHERE + issues.project_id = work_items.project_id + AND issues.iid = CASE WHEN system_note_metadata.action='relate_to_parent' THEN substring(notes.note from 'added #(\\d+) as parent')::bigint + ELSE substring(notes.note from 'removed parent \\S+ #(\\d+)')::bigint + END + ) parent_issues + WHERE + system_note_metadata.id IN (#{ids_subquery}) + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb b/lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb new file mode 100644 index 00000000000..23c510720c0 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_root_storage_statistics_fork_storage_sizes.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill the following columns on the namespace_root_storage_statistics table: + # - public_forks_storage_size + # - internal_forks_storage_size + # - private_forks_storage_size + class BackfillRootStorageStatisticsForkStorageSizes < BatchedMigrationJob + operation_name :backfill_root_storage_statistics_fork_sizes + feature_category :consumables_cost_management + + VISIBILITY_LEVELS_TO_STORAGE_SIZE_COLUMNS = { + 0 => :private_forks_storage_size, + 10 => :internal_forks_storage_size, + 20 => :public_forks_storage_size + }.freeze + + def perform + each_sub_batch do |sub_batch| + sub_batch.each do |root_storage_statistics| + next if has_fork_data?(root_storage_statistics) + + namespace_id = root_storage_statistics.namespace_id + + namespace_type = execute("SELECT type FROM namespaces WHERE id = #{namespace_id}").first&.fetch('type') + + next if namespace_type.nil? + + sql = if user_namespace?(namespace_type) + user_namespace_sql(namespace_id) + else + group_namespace_sql(namespace_id) + end + + stats = execute(sql) + .map { |h| { h['projects_visibility_level'] => h['sum_project_statistics_storage_size'] } } + .reduce({}) { |memo, h| memo.merge(h) } + .transform_keys { |k| VISIBILITY_LEVELS_TO_STORAGE_SIZE_COLUMNS[k] } + + root_storage_statistics.update!(stats) + end + end + end + + def has_fork_data?(root_storage_statistics) + root_storage_statistics.public_forks_storage_size != 0 || + root_storage_statistics.internal_forks_storage_size != 0 || + root_storage_statistics.private_forks_storage_size != 0 + end + + def user_namespace?(type) + type.nil? || type == 'User' || !(type == 'Group' || type == 'Project') + end + + def execute(sql) + ::ApplicationRecord.connection.execute(sql) + end + + def user_namespace_sql(namespace_id) + <<~SQL + SELECT + SUM("project_statistics"."storage_size") AS sum_project_statistics_storage_size, + "projects"."visibility_level" AS projects_visibility_level + FROM + "projects" + INNER JOIN "project_statistics" ON "project_statistics"."project_id" = "projects"."id" + INNER JOIN "fork_network_members" ON "fork_network_members"."project_id" = "projects"."id" + INNER JOIN "fork_networks" ON "fork_networks"."id" = "fork_network_members"."fork_network_id" + WHERE + "projects"."namespace_id" = #{namespace_id} + AND (fork_networks.root_project_id != projects.id) + GROUP BY "projects"."visibility_level" + SQL + end + + def group_namespace_sql(namespace_id) + <<~SQL + SELECT + SUM("project_statistics"."storage_size") AS sum_project_statistics_storage_size, + "projects"."visibility_level" AS projects_visibility_level + FROM + "projects" + INNER JOIN "project_statistics" ON "project_statistics"."project_id" = "projects"."id" + INNER JOIN "fork_network_members" ON "fork_network_members"."project_id" = "projects"."id" + INNER JOIN "fork_networks" ON "fork_networks"."id" = "fork_network_members"."fork_network_id" + WHERE + "projects"."namespace_id" IN ( + SELECT namespaces.traversal_ids[array_length(namespaces.traversal_ids, 1)] AS id + FROM "namespaces" + WHERE "namespaces"."type" = 'Group' AND (traversal_ids @> ('{#{namespace_id}}')) + ) + AND (fork_networks.root_project_id != projects.id) + GROUP BY "projects"."visibility_level" + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb b/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb deleted file mode 100644 index b703faf6a6c..00000000000 --- a/lib/gitlab/background_migration/cleanup_draft_data_from_faulty_regex.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Cleanup draft column data inserted by a faulty regex - # - class CleanupDraftDataFromFaultyRegex - # Migration only version of MergeRequest table - ## - class MergeRequest < ActiveRecord::Base - LEAKY_REGEXP_STR = "^\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP" - CORRECTED_REGEXP_STR = "^(\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP)" - - include EachBatch - - self.table_name = 'merge_requests' - - def self.eligible - where(state_id: 1) - .where(draft: true) - .where("title ~* ?", LEAKY_REGEXP_STR) - .where("title !~* ?", CORRECTED_REGEXP_STR) - end - end - - def perform(start_id, end_id) - eligible_mrs = MergeRequest.eligible.where(id: start_id..end_id).pluck(:id) - - return if eligible_mrs.empty? - - eligible_mrs.each_slice(10) do |slice| - MergeRequest.where(id: slice).update_all(draft: false) - end - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'CleanupDraftDataFromFaultyRegex', - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb deleted file mode 100644 index 28c28ae48eb..00000000000 --- a/lib/gitlab/background_migration/encrypt_integration_properties.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Migrates the integration.properties column from plaintext to encrypted text. - class EncryptIntegrationProperties - # The Integration model, with just the relevant bits. - class Integration < ActiveRecord::Base - include EachBatch - - ALGORITHM = 'aes-256-gcm' - - self.table_name = 'integrations' - self.inheritance_column = :_type_disabled - - scope :with_properties, -> { where.not(properties: nil) } - scope :not_already_encrypted, -> { where(encrypted_properties: nil) } - scope :for_batch, ->(range) { where(id: range) } - - attr_encrypted :encrypted_properties_tmp, - attribute: :encrypted_properties, - mode: :per_attribute_iv, - key: ::Settings.attr_encrypted_db_key_base_32, - algorithm: ALGORITHM, - marshal: true, - marshaler: ::Gitlab::Json, - encode: false, - encode_iv: false - - # See 'Integration#reencrypt_properties' - def encrypt_properties - data = ::Gitlab::Json.parse(properties) - iv = generate_iv(ALGORITHM) - ep = self.class.attr_encrypt(:encrypted_properties_tmp, data, { iv: iv }) - - [ep, iv] - end - end - - def perform(start_id, stop_id) - batch_query = Integration.with_properties.not_already_encrypted.for_batch(start_id..stop_id) - encrypt_batch(batch_query) - mark_job_as_succeeded(start_id, stop_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - - # represent binary string as a PSQL binary literal: - # https://www.postgresql.org/docs/9.4/datatype-binary.html - def bytea(value) - "'\\x#{value.unpack1('H*')}'::bytea" - end - - def encrypt_batch(batch_query) - values = batch_query.select(:id, :properties).map do |record| - encrypted_properties, encrypted_properties_iv = record.encrypt_properties - "(#{record.id}, #{bytea(encrypted_properties)}, #{bytea(encrypted_properties_iv)})" - end - - return if values.empty? - - Integration.connection.execute(<<~SQL.squish) - WITH cte(cte_id, cte_encrypted_properties, cte_encrypted_properties_iv) - AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT * - FROM (VALUES #{values.join(',')}) AS t (id, encrypted_properties, encrypted_properties_iv) - ) - UPDATE #{Integration.table_name} - SET encrypted_properties = cte_encrypted_properties - , encrypted_properties_iv = cte_encrypted_properties_iv - FROM cte - WHERE cte_id = id - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/encrypt_static_object_token.rb b/lib/gitlab/background_migration/encrypt_static_object_token.rb deleted file mode 100644 index 961dea028c9..00000000000 --- a/lib/gitlab/background_migration/encrypt_static_object_token.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Populates "static_object_token_encrypted" field with encrypted versions - # of values from "static_object_token" field - class EncryptStaticObjectToken - # rubocop:disable Style/Documentation - class User < ActiveRecord::Base - include ::EachBatch - self.table_name = 'users' - scope :with_static_object_token, -> { where.not(static_object_token: nil) } - scope :without_static_object_token_encrypted, -> { where(static_object_token_encrypted: nil) } - end - # rubocop:enable Style/Documentation - - BATCH_SIZE = 100 - - def perform(start_id, end_id) - ranged_query = User - .where(id: start_id..end_id) - .with_static_object_token - .without_static_object_token_encrypted - - ranged_query.each_batch(of: BATCH_SIZE) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) - - batch_query = User.unscoped - .where(id: first..last) - .with_static_object_token - .without_static_object_token_encrypted - - user_tokens = batch_query.pluck(:id, :static_object_token) - - user_encrypted_tokens = user_tokens.map do |(id, plaintext_token)| - next if plaintext_token.blank? - - [id, Gitlab::CryptoHelper.aes256_gcm_encrypt(plaintext_token)] - end - - encrypted_tokens_sql = user_encrypted_tokens.compact.map { |(id, token)| "(#{id}, '#{token}')" }.join(',') - - next unless user_encrypted_tokens.present? - - User.connection.execute(<<~SQL) - WITH cte(cte_id, cte_token) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( - SELECT * - FROM (VALUES #{encrypted_tokens_sql}) AS t (id, token) - ) - UPDATE #{User.table_name} - SET static_object_token_encrypted = cte_token - FROM cte - WHERE cte_id = id - SQL - end - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb b/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb deleted file mode 100644 index 3772430d0b7..00000000000 --- a/lib/gitlab/background_migration/fix_duplicate_project_name_and_path.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Fix project name duplicates and backfill missing project namespace ids - class FixDuplicateProjectNameAndPath - SUB_BATCH_SIZE = 10 - # isolated project active record - class Project < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'projects' - - scope :without_project_namespace, -> { where(project_namespace_id: nil) } - scope :id_in, ->(ids) { where(id: ids) } - end - - def perform(start_id, end_id) - @project_ids = fetch_project_ids(start_id, end_id) - backfill_project_namespaces_service = init_backfill_service(project_ids) - backfill_project_namespaces_service.cleanup_gin_index('projects') - - project_ids.each_slice(SUB_BATCH_SIZE) do |ids| - ApplicationRecord.connection.execute(update_projects_name_and_path_sql(ids)) - end - - backfill_project_namespaces_service.backfill_project_namespaces - - mark_job_as_succeeded(start_id, end_id) - end - - private - - attr_accessor :project_ids - - def fetch_project_ids(start_id, end_id) - Project.without_project_namespace.where(id: start_id..end_id) - end - - def init_backfill_service(project_ids) - service = Gitlab::BackgroundMigration::ProjectNamespaces::BackfillProjectNamespaces.new - service.project_ids = project_ids - service.sub_batch_size = SUB_BATCH_SIZE - - service - end - - def update_projects_name_and_path_sql(project_ids) - <<~SQL - WITH cte (project_id, path_from_route ) AS ( - #{path_from_route_sql(project_ids).to_sql} - ) - UPDATE - projects - SET - name = concat(projects.name, '-', id), - path = CASE - WHEN projects.path <> cte.path_from_route THEN path_from_route - ELSE projects.path - END - FROM - cte - WHERE - projects.id = cte.project_id; - SQL - end - - def path_from_route_sql(project_ids) - Project.without_project_namespace.id_in(project_ids) - .joins("INNER JOIN routes ON routes.source_id = projects.id AND routes.source_type = 'Project'") - .select("projects.id, SUBSTRING(routes.path FROM '[^/]+(?=/$|$)') AS path_from_route") - end - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'FixDuplicateProjectNameAndPath', - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb b/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb deleted file mode 100644 index 2c09b8c0b24..00000000000 --- a/lib/gitlab/background_migration/fix_incorrect_max_seats_used.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class FixIncorrectMaxSeatsUsed - def perform(batch = nil) - end - end - end -end - -Gitlab::BackgroundMigration::FixIncorrectMaxSeatsUsed.prepend_mod_with('Gitlab::BackgroundMigration::FixIncorrectMaxSeatsUsed') diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb deleted file mode 100644 index db3f98bc2ba..00000000000 --- a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb +++ /dev/null @@ -1,124 +0,0 @@ -# frozen_string_literal: true - -require 'parser/ruby27' - -module Gitlab - module BackgroundMigration - # This migration fixes raw_metadata entries which have incorrectly been passed a Ruby Hash instead of JSON data. - class FixVulnerabilityOccurrencesWithHashesAsRawMetadata - CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7 - GENERIC_REPORT_TYPE = 99 - - # Type error is used to handle unexpected types when parsing stringified hashes. - class TypeError < ::StandardError - attr_reader :message, :type - - def initialize(message, type) - @message = message - @type = type - end - end - - # Migration model namespace isolated from application code. - class Finding < ActiveRecord::Base - include EachBatch - - self.table_name = 'vulnerability_occurrences' - - scope :by_api_report_types, -> { where(report_type: [CLUSTER_IMAGE_SCANNING_REPORT_TYPE, GENERIC_REPORT_TYPE]) } - end - - def perform(start_id, end_id) - Finding.by_api_report_types.where(id: start_id..end_id).each do |finding| - next if valid_json?(finding.raw_metadata) - - metadata = hash_from_s(finding.raw_metadata) - - finding.update(raw_metadata: metadata.to_json) if metadata - end - mark_job_as_succeeded(start_id, end_id) - end - - def hash_from_s(str_hash) - ast = Parser::Ruby27.parse(str_hash) - - unless ast.type == :hash - ::Gitlab::AppLogger.error(message: "expected raw_metadata to be a hash", type: ast.type) - return - end - - parse_hash(ast) - rescue Parser::SyntaxError => e - ::Gitlab::AppLogger.error(message: "error parsing raw_metadata", error: e.message) - nil - rescue TypeError => e - ::Gitlab::AppLogger.error(message: "error parsing raw_metadata", error: e.message, type: e.type) - nil - end - - private - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'FixVulnerabilityOccurrencesWithHashesAsRawMetadata', - arguments - ) - end - - def valid_json?(metadata) - Oj.load(metadata) - true - rescue Oj::ParseError, EncodingError, JSON::ParserError, JSON::GeneratorError, Encoding::UndefinedConversionError - false - end - - def parse_hash(hash) - out = {} - hash.children.each do |node| - unless node.type == :pair - raise TypeError.new("expected child of hash to be a `pair`", node.type) - end - - key, value = node.children - - key = parse_key(key) - value = parse_value(value) - - out[key] = value - end - - out - end - - def parse_key(key) - case key.type - when :sym, :str, :int - key.children.first - else - raise TypeError.new("expected key to be either symbol, string, or integer", key.type) - end - end - - def parse_value(value) - case value.type - when :sym, :str, :int - value.children.first - # rubocop:disable Lint/BooleanSymbol - when :true - true - when :false - false - # rubocop:enable Lint/BooleanSymbol - when :nil - nil - when :array - value.children.map { |c| parse_value(c) } - when :hash - parse_hash(value) - else - raise TypeError.new("value of a pair was an unexpected type", value.type) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction.rb b/lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction.rb new file mode 100644 index 00000000000..68cc650d130 --- /dev/null +++ b/lib/gitlab/background_migration/mark_duplicate_npm_packages_for_destruction.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # It seeks duplicate npm packages and mark them for destruction + class MarkDuplicateNpmPackagesForDestruction < BatchedMigrationJob + NPM_PACKAGE_TYPE = 2 + PENDING_DESTRUCTION_STATUS = 4 + + operation_name :update_all + feature_category :package_registry + + # Temporary class to link AR model to the `packages_packages` table + class Package < ::ApplicationRecord + include EachBatch + + self.table_name = 'packages_packages' + end + + def perform + distinct_each_batch do |batch| + project_ids = batch.pluck(:project_id) + + subquery = Package + .where(project_id: project_ids, package_type: NPM_PACKAGE_TYPE) + .where.not(status: PENDING_DESTRUCTION_STATUS) + .select('project_id, name, version, MAX(id) AS max_id') + .group(:project_id, :name, :version) + .having('COUNT(*) > 1') + + join_query = <<~SQL.squish + INNER JOIN (#{subquery.to_sql}) AS duplicates + ON packages_packages.project_id = duplicates.project_id + AND packages_packages.name = duplicates.name + AND packages_packages.version = duplicates.version + SQL + + Package + .joins(join_query) + .where.not('packages_packages.id = duplicates.max_id') + .each_batch do |batch_to_update| + batch_to_update.update_all(status: PENDING_DESTRUCTION_STATUS) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/merge_topics_with_same_name.rb b/lib/gitlab/background_migration/merge_topics_with_same_name.rb deleted file mode 100644 index 07231098a5f..00000000000 --- a/lib/gitlab/background_migration/merge_topics_with_same_name.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to merge project topics with the same case insensitive name - class MergeTopicsWithSameName - # Temporary AR model for topics - class Topic < ActiveRecord::Base - self.table_name = 'topics' - end - - # Temporary AR model for project topic assignment - class ProjectTopic < ActiveRecord::Base - self.table_name = 'project_topics' - end - - def perform(topic_names) - topic_names.each do |topic_name| - topics = Topic.where('LOWER(name) = ?', topic_name) - .order(total_projects_count: :desc, non_private_projects_count: :desc, id: :asc) - .to_a - topic_to_keep = topics.shift - merge_topics(topic_to_keep, topics) if topics.any? - end - end - - private - - def merge_topics(topic_to_keep, topics_to_remove) - description = topic_to_keep.description - - topics_to_remove.each do |topic| - description ||= topic.description if topic.description.present? - process_avatar(topic_to_keep, topic) if topic.avatar.present? - - ProjectTopic.transaction do - ProjectTopic.where(topic_id: topic.id) - .where.not(project_id: ProjectTopic.where(topic_id: topic_to_keep).select(:project_id)) - .update_all(topic_id: topic_to_keep.id) - ProjectTopic.where(topic_id: topic.id).delete_all - end - end - - Topic.where(id: topics_to_remove).delete_all - - topic_to_keep.update( - description: description, - total_projects_count: total_projects_count(topic_to_keep.id), - non_private_projects_count: non_private_projects_count(topic_to_keep.id) - ) - end - - # We intentionally use application code here because we need to copy/remove avatar files - def process_avatar(topic_to_keep, topic_to_remove) - topic_to_remove = ::Projects::Topic.find(topic_to_remove.id) - topic_to_keep = ::Projects::Topic.find(topic_to_keep.id) - unless topic_to_keep.avatar.present? - topic_to_keep.avatar = topic_to_remove.avatar - topic_to_keep.save! - end - - topic_to_remove.remove_avatar! - topic_to_remove.save! - end - - def total_projects_count(topic_id) - ProjectTopic.where(topic_id: topic_id).count - end - - def non_private_projects_count(topic_id) - ProjectTopic.joins('INNER JOIN projects ON project_topics.project_id = projects.id') - .where(project_topics: { topic_id: topic_id }).where('projects.visibility_level in (10, 20)').count - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb b/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb deleted file mode 100644 index 49eff6e2771..00000000000 --- a/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Migrates personal namespace project `maintainer` memberships (for the associated user only) to OWNER - # Does not create any missing records, simply migrates existing ones - class MigratePersonalNamespaceProjectMaintainerToOwner - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:update_all) do - sub_batch.update_all('access_level = 50') - end - - pause_ms = 0 if pause_ms < 0 - sleep(pause_ms * 0.001) - end - end - - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - - private - - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - # members of projects within their own personal namespace - - # rubocop: disable CodeReuse/ActiveRecord - define_batchable_model(:members, connection: ApplicationRecord.connection) - .where(source_key_column => start_id..stop_id) - .joins("INNER JOIN projects ON members.source_id = projects.id") - .joins("INNER JOIN namespaces ON projects.namespace_id = namespaces.id") - .where(type: 'ProjectMember') - .where("namespaces.type = 'User'") - .where('members.access_level < 50') - .where('namespaces.owner_id = members.user_id') - end - end - # rubocop: enable CodeReuse/ActiveRecord - end -end diff --git a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb b/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb deleted file mode 100644 index d7d24960a41..00000000000 --- a/lib/gitlab/background_migration/migrate_shimo_confluence_integration_category.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to migrate category of integrations to third_party_wiki for confluence and shimo - class MigrateShimoConfluenceIntegrationCategory - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id) - define_batchable_model('integrations', connection: ApplicationRecord.connection) - .where(id: start_id..end_id, type_new: %w[Integrations::Confluence Integrations::Shimo]) - .update_all(category: 'third_party_wiki') - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb index 592ef3220ff..74f5bc3f725 100644 --- a/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb +++ b/lib/gitlab/background_migration/nullify_creator_id_column_of_orphaned_projects.rb @@ -12,7 +12,7 @@ module Gitlab end operation_name :update_all - feature_category :projects + feature_category :groups_and_projects def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb deleted file mode 100644 index 13b66b2e02e..00000000000 --- a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to nullify orphan runner_id on ci_builds table - class NullifyOrphanRunnerIdOnCiBuilds - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) - pause_ms = 0 if pause_ms < 0 - - batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - batch_metrics.time_operation(:update_all) do - filtered_sub_batch(sub_batch).update_all(runner_id: nil) - end - - sleep(pause_ms * 0.001) - end - end - - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - - private - - def connection - ::Ci::ApplicationRecord.connection - end - - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: connection) - .where(source_key_column => start_id..stop_id) - end - - def filtered_sub_batch(sub_batch) - sub_batch - .joins('LEFT OUTER JOIN ci_runners ON ci_runners.id = ci_builds.runner_id') - .where('ci_builds.runner_id IS NOT NULL AND ci_runners.id IS NULL') - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb b/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb deleted file mode 100644 index a9611e9814c..00000000000 --- a/lib/gitlab/background_migration/populate_container_repository_migration_plan.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to populates the migration_plan column of container_repositories - # with the current plan of the namespaces that owns the container_repository - # - # The plan can be NULL, in which case no UPDATE - # will be executed. - class PopulateContainerRepositoryMigrationPlan - def perform(start_id, end_id) - (start_id..end_id).each do |id| - execute(<<~SQL) - WITH selected_plan AS ( - SELECT "plans"."name" - FROM "container_repositories" - INNER JOIN "projects" ON "projects"."id" = "container_repositories"."project_id" - INNER JOIN "namespaces" ON "namespaces"."id" = "projects"."namespace_id" - INNER JOIN "gitlab_subscriptions" ON "gitlab_subscriptions"."namespace_id" = "namespaces"."traversal_ids"[1] - INNER JOIN "plans" ON "plans"."id" = "gitlab_subscriptions"."hosted_plan_id" - WHERE "container_repositories"."id" = #{id} - ) - UPDATE container_repositories - SET migration_plan = selected_plan.name - FROM selected_plan - WHERE container_repositories.id = #{id}; - SQL - end - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def connection - @connection ||= ApplicationRecord.connection - end - - def execute(sql) - connection.execute(sql) - end - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_namespace_statistics.rb b/lib/gitlab/background_migration/populate_namespace_statistics.rb deleted file mode 100644 index 97927ef48c2..00000000000 --- a/lib/gitlab/background_migration/populate_namespace_statistics.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This class creates/updates those namespace statistics - # that haven't been created nor initialized. - # It also updates the related namespace statistics - class PopulateNamespaceStatistics - def perform(group_ids, statistics) - # Updating group statistics might involve calling Gitaly. - # For example, when calculating `wiki_size`, we will need - # to perform the request to check if the repo exists and - # also the repository size. - # - # The `allow_n_plus_1_calls` method is only intended for - # dev and test. It won't be raised in prod. - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - relation(group_ids).each do |group| - upsert_namespace_statistics(group, statistics) - end - end - end - - private - - def upsert_namespace_statistics(group, statistics) - response = ::Groups::UpdateStatisticsService.new(group, statistics: statistics).execute - - error_message("#{response.message} group: #{group.id}") if response.error? - end - - def logger - @logger ||= ::Gitlab::BackgroundMigration::Logger.build - end - - def error_message(message) - logger.error(message: "Namespace Statistics Migration: #{message}") - end - - def relation(group_ids) - Group.includes(:namespace_statistics).where(id: group_ids) - end - end - end -end - -Gitlab::BackgroundMigration::PopulateNamespaceStatistics.prepend_mod_with('Gitlab::BackgroundMigration::PopulateNamespaceStatistics') diff --git a/lib/gitlab/background_migration/populate_test_reports_issue_id.rb b/lib/gitlab/background_migration/populate_test_reports_issue_id.rb deleted file mode 100644 index 301efd0c943..00000000000 --- a/lib/gitlab/background_migration/populate_test_reports_issue_id.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true -# rubocop: disable Style/Documentation - -module Gitlab - module BackgroundMigration - class PopulateTestReportsIssueId - def perform(start_id, stop_id) - # NO OP - end - end - end -end - -Gitlab::BackgroundMigration::PopulateTestReportsIssueId.prepend_mod diff --git a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb b/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb deleted file mode 100644 index 1f2b55004e4..00000000000 --- a/lib/gitlab/background_migration/populate_topics_non_private_projects_count.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to populates the non private projects counter of topics - class PopulateTopicsNonPrivateProjectsCount - SUB_BATCH_SIZE = 100 - - # Temporary AR model for topics - class Topic < ActiveRecord::Base - include EachBatch - - self.table_name = 'topics' - end - - def perform(start_id, stop_id) - Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch| - ApplicationRecord.connection.execute(<<~SQL) - WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql}) - UPDATE topics - SET non_private_projects_count = ( - SELECT COUNT(*) - FROM project_topics - INNER JOIN projects - ON project_topics.project_id = projects.id - WHERE project_topics.topic_id = batched_relation.id - AND projects.visibility_level > 0 - ) - FROM batched_relation - WHERE topics.id = batched_relation.id - SQL - end - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_vulnerability_reads.rb b/lib/gitlab/background_migration/populate_vulnerability_reads.rb deleted file mode 100644 index 656c62d9ee5..00000000000 --- a/lib/gitlab/background_migration/populate_vulnerability_reads.rb +++ /dev/null @@ -1,84 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop:disable Style/Documentation - class PopulateVulnerabilityReads - include Gitlab::Database::DynamicModelHelpers - - PAUSE_SECONDS = 0.1 - - def perform(start_id, end_id, sub_batch_size) - vulnerability_model.where(id: start_id..end_id).each_batch(of: sub_batch_size) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) - connection.execute(insert_query(first, last)) - - sleep PAUSE_SECONDS - end - - mark_job_as_succeeded(start_id, end_id, sub_batch_size) - end - - private - - def vulnerability_model - define_batchable_model('vulnerabilities', connection: connection) - end - - def connection - ApplicationRecord.connection - end - - def insert_query(start_id, end_id) - <<~SQL - INSERT INTO vulnerability_reads ( - vulnerability_id, - project_id, - scanner_id, - report_type, - severity, - state, - has_issues, - resolved_on_default_branch, - uuid, - location_image - ) - SELECT - vulnerabilities.id, - vulnerabilities.project_id, - vulnerability_scanners.id, - vulnerabilities.report_type, - vulnerabilities.severity, - vulnerabilities.state, - CASE - WHEN - vulnerability_issue_links.vulnerability_id IS NOT NULL - THEN - true - ELSE - false - END - has_issues, - vulnerabilities.resolved_on_default_branch, - vulnerability_occurrences.uuid::uuid, - vulnerability_occurrences.location ->> 'image' - FROM - vulnerabilities - INNER JOIN vulnerability_occurrences ON vulnerability_occurrences.vulnerability_id = vulnerabilities.id - INNER JOIN vulnerability_scanners ON vulnerability_scanners.id = vulnerability_occurrences.scanner_id - LEFT JOIN vulnerability_issue_links ON vulnerability_issue_links.vulnerability_id = vulnerabilities.id - WHERE vulnerabilities.id BETWEEN #{start_id} AND #{end_id} - ON CONFLICT(vulnerability_id) DO NOTHING; - SQL - end - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - # rubocop:enable Style/Documentation - end -end diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb deleted file mode 100644 index 9a42d035285..00000000000 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ /dev/null @@ -1,218 +0,0 @@ -# frozen_string_literal: true - -# rubocop: disable Style/Documentation -class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # rubocop:disable Metrics/ClassLength - # rubocop: disable Gitlab/NamespacedClass - class VulnerabilitiesIdentifier < ActiveRecord::Base - self.table_name = "vulnerability_identifiers" - has_many :primary_findings, class_name: 'VulnerabilitiesFinding', inverse_of: :primary_identifier, foreign_key: 'primary_identifier_id' - end - - class VulnerabilitiesFinding < ActiveRecord::Base - include EachBatch - include ShaAttribute - - self.table_name = "vulnerability_occurrences" - - has_many :signatures, foreign_key: 'finding_id', class_name: 'VulnerabilityFindingSignature', inverse_of: :finding - belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id' - - REPORT_TYPES = { - sast: 0, - dependency_scanning: 1, - container_scanning: 2, - dast: 3, - secret_detection: 4, - coverage_fuzzing: 5, - api_fuzzing: 6, - cluster_image_scanning: 7, - generic: 99 - }.with_indifferent_access.freeze - enum report_type: REPORT_TYPES - - sha_attribute :fingerprint - sha_attribute :location_fingerprint - end - - class VulnerabilityFindingSignature < ActiveRecord::Base - include ShaAttribute - - self.table_name = 'vulnerability_finding_signatures' - belongs_to :finding, foreign_key: 'finding_id', inverse_of: :signatures, class_name: 'VulnerabilitiesFinding' - - sha_attribute :signature_sha - end - - class VulnerabilitiesFindingPipeline < ActiveRecord::Base - include EachBatch - self.table_name = "vulnerability_occurrence_pipelines" - end - - class Vulnerability < ActiveRecord::Base - include EachBatch - self.table_name = "vulnerabilities" - end - - class CalculateFindingUUID - FINDING_NAMESPACES_IDS = { - development: "a143e9e2-41b3-47bc-9a19-081d089229f4", - test: "a143e9e2-41b3-47bc-9a19-081d089229f4", - staging: "a6930898-a1b2-4365-ab18-12aa474d9b26", - production: "58dc0f06-936c-43b3-93bb-71693f1b6570" - }.freeze - - NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze - PACK_PATTERN = "NnnnnN" - - def self.call(value) - Digest::UUID.uuid_v5(namespace_id, value) - end - - def self.namespace_id - namespace_uuid = FINDING_NAMESPACES_IDS.fetch(Rails.env.to_sym) - # Digest::UUID is broken when using an UUID in namespace_id - # https://github.com/rails/rails/issues/37681#issue-520718028 - namespace_uuid.scan(NAMESPACE_REGEX).flatten.map { |s| s.to_i(16) }.pack(PACK_PATTERN) - end - end - # rubocop: enable Gitlab/NamespacedClass - - # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength - def perform(start_id, end_id) - log_info('Migration started', start_id: start_id, end_id: end_id) - - VulnerabilitiesFinding - .joins(:primary_identifier) - .includes(:signatures) - .select(:id, :report_type, :primary_identifier_id, :fingerprint, :location_fingerprint, :project_id, :created_at, :vulnerability_id, :uuid) - .where(id: start_id..end_id) - .each_batch(of: 50) do |relation| - duplicates = find_duplicates(relation) - remove_findings(ids: duplicates) if duplicates.present? - - to_update = relation.reject { |finding| duplicates.include?(finding.id) } - - begin - known_uuids = Set.new - to_be_deleted = [] - - mappings = to_update.each_with_object({}) do |finding, hash| - uuid = calculate_uuid_v5_for_finding(finding) - - if known_uuids.add?(uuid) - hash[finding] = { uuid: uuid } - else - to_be_deleted << finding.id - end - end - - # It is technically still possible to have duplicate uuids - # if the data integrity is broken somehow and the primary identifiers of - # the findings are pointing to different projects with the same fingerprint values. - if to_be_deleted.present? - log_info('Conflicting UUIDs found within the batch', finding_ids: to_be_deleted) - - remove_findings(ids: to_be_deleted) - end - - ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings) if mappings.present? - - log_info('Recalculation is done', finding_ids: mappings.keys.pluck(:id)) - rescue ActiveRecord::RecordNotUnique => error - log_info('RecordNotUnique error received') - - match_data = /\(uuid\)=\((?<uuid>\S{36})\)/.match(error.message) - - # This exception returns the **correct** UUIDv5 which probably comes from a later record - # and it's the one we can drop in the easiest way before retrying the UPDATE query - if match_data - uuid = match_data[:uuid] - log_info('Conflicting UUID found', uuid: uuid) - - id = VulnerabilitiesFinding.find_by(uuid: uuid)&.id - remove_findings(ids: id) if id - retry - else - log_error('Couldnt find conflicting uuid') - - Gitlab::ErrorTracking.track_and_raise_exception(error) - end - end - end - - mark_job_as_succeeded(start_id, end_id) - rescue StandardError => error - log_error('An exception happened') - - Gitlab::ErrorTracking.track_and_raise_exception(error) - end - # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength - - private - - def find_duplicates(relation) - to_exclude = [] - relation.flat_map do |record| - # Assuming we're scanning id 31 and the duplicate is id 40 - # first we'd process 31 and add 40 to the list of ids to remove - # then we would process record 40 and add 31 to the list of removals - # so we would drop both records - to_exclude << record.id - - VulnerabilitiesFinding.where( - report_type: record.report_type, - location_fingerprint: record.location_fingerprint, - primary_identifier_id: record.primary_identifier_id, - project_id: record.project_id - ).where.not(id: to_exclude).pluck(:id) - end - end - - def remove_findings(ids:) - ids = Array(ids) - log_info('Removing Findings and associated records', ids: ids) - - vulnerability_ids = VulnerabilitiesFinding.where(id: ids).pluck(:vulnerability_id).uniq.compact - - VulnerabilitiesFindingPipeline.where(occurrence_id: ids).each_batch { |batch| batch.delete_all } - Vulnerability.where(id: vulnerability_ids).each_batch { |batch| batch.delete_all } - VulnerabilitiesFinding.where(id: ids).delete_all - end - - def calculate_uuid_v5_for_finding(vulnerability_finding) - return unless vulnerability_finding - - signatures = vulnerability_finding.signatures.sort_by { |signature| signature.algorithm_type_before_type_cast } - location_fingerprint = signatures.last&.signature_sha || vulnerability_finding.location_fingerprint - - uuid_v5_name_components = { - report_type: vulnerability_finding.report_type, - primary_identifier_fingerprint: vulnerability_finding.fingerprint, - location_fingerprint: location_fingerprint, - project_id: vulnerability_finding.project_id - } - - name = uuid_v5_name_components.values.join('-') - - CalculateFindingUUID.call(name) - end - - def log_info(message, **extra) - logger.info(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra) - end - - def log_error(message, **extra) - logger.error(migrator: 'RecalculateVulnerabilitiesOccurrencesUuid', message: message, **extra) - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'RecalculateVulnerabilitiesOccurrencesUuid', - arguments - ) - end -end diff --git a/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb b/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb deleted file mode 100644 index 20200a1d508..00000000000 --- a/lib/gitlab/background_migration/recalculate_vulnerability_finding_signatures_for_findings.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class RecalculateVulnerabilityFindingSignaturesForFindings - def perform(start_id, stop_id) - end - end - end -end - -Gitlab::BackgroundMigration::RecalculateVulnerabilityFindingSignaturesForFindings.prepend_mod diff --git a/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb b/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb deleted file mode 100644 index d47aa76f24b..00000000000 --- a/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Removing expire_at timestamps that shouldn't have - # been written to traces on gitlab.com. - class RemoveAllTraceExpirationDates - include Gitlab::Database::MigrationHelpers - - BATCH_SIZE = 1_000 - - # Stubbed class to connect to the CI database - # connects_to has to be called in abstract classes. - class MultiDbAdaptableClass < ActiveRecord::Base - self.abstract_class = true - - if Gitlab::Database.has_config?(:ci) - connects_to database: { writing: :ci, reading: :ci } - end - end - - # Stubbed class to access the ci_job_artifacts table - class JobArtifact < MultiDbAdaptableClass - include EachBatch - - self.table_name = 'ci_job_artifacts' - - TARGET_TIMESTAMPS = [ - Date.new(2021, 04, 22).midnight.utc, - Date.new(2021, 05, 22).midnight.utc, - Date.new(2021, 06, 22).midnight.utc, - Date.new(2022, 01, 22).midnight.utc, - Date.new(2022, 02, 22).midnight.utc, - Date.new(2022, 03, 22).midnight.utc, - Date.new(2022, 04, 22).midnight.utc - ].freeze - - scope :traces, -> { where(file_type: 3) } - scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } - scope :in_targeted_timestamps, -> { where(expire_at: TARGET_TIMESTAMPS) } - end - - def perform(start_id, end_id) - return unless Gitlab.com? - - JobArtifact.traces - .between(start_id, end_id) - .in_targeted_timestamps - .each_batch(of: BATCH_SIZE) { |batch| batch.update_all(expire_at: nil) } - end - end - end -end diff --git a/lib/gitlab/background_migration/remove_invalid_deploy_access_level_groups.rb b/lib/gitlab/background_migration/remove_invalid_deploy_access_level_groups.rb new file mode 100644 index 00000000000..0a107a136b0 --- /dev/null +++ b/lib/gitlab/background_migration/remove_invalid_deploy_access_level_groups.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class removes invalid `protected_environment_deploy_access_levels.group_id` records. + class RemoveInvalidDeployAccessLevelGroups < BatchedMigrationJob + operation_name :remove_invalid_deploy_access_level_groups + feature_category :database + + scope_to ->(relation) do + relation.joins('INNER JOIN namespaces ON namespaces.id = protected_environment_deploy_access_levels.group_id') + .where.not(protected_environment_deploy_access_levels: { group_id: nil }) + .where("namespaces.type = 'User'") + end + + def perform + each_sub_batch(&:delete_all) + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb b/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb index 879e52c96bf..713131edd30 100644 --- a/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb +++ b/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb @@ -7,7 +7,7 @@ module Gitlab class RemoveProjectGroupLinkWithMissingGroups < Gitlab::BackgroundMigration::BatchedMigrationJob scope_to ->(relation) { relation } operation_name :delete_all - feature_category :subgroups + feature_category :groups_and_projects def perform each_sub_batch do |sub_batch| diff --git a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb b/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb deleted file mode 100644 index 4acef9029f9..00000000000 --- a/lib/gitlab/background_migration/remove_vulnerability_finding_links.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Remove vulnerability finding link records - # The records will be repopulated from the `raw_metadata` - # column of `vulnerability_occurrences` once the unique - # index is in place. - class RemoveVulnerabilityFindingLinks - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, stop_id) - define_batchable_model('vulnerability_finding_links', connection: ApplicationRecord.connection) - .where(id: start_id..stop_id) - .delete_all - end - end - end -end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb deleted file mode 100644 index 190e2fc22fb..00000000000 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to nullify duplicate runners_token_encrypted values in projects table in batches - class ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects - class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'projects' - - scope :base_query, -> { where.not(runners_token_encrypted: nil) } - end - - def perform(start_id, end_id) - # Reset duplicate runner tokens that would prevent creating an unique index. - batch_records = Project.base_query.where(id: start_id..end_id) - - duplicate_tokens = Project.base_query - .where(runners_token_encrypted: batch_records.select(:runners_token_encrypted).distinct) - .group(:runners_token_encrypted) - .having('COUNT(*) > 1') - .pluck(:runners_token_encrypted) - - batch_records.where(runners_token_encrypted: duplicate_tokens).update_all(runners_token_encrypted: nil) if duplicate_tokens.any? - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb deleted file mode 100644 index b58eefa0ab3..00000000000 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to nullify duplicate ci_runners_token values in projects table in batches - class ResetDuplicateCiRunnersTokenValuesOnProjects - class Project < ActiveRecord::Base # rubocop:disable Style/Documentation - include EachBatch - - self.table_name = 'projects' - - scope :base_query, -> { where.not(runners_token: nil) } - end - - def perform(start_id, end_id) - # Reset duplicate runner tokens that would prevent creating an unique index. - batch_records = Project.base_query.where(id: start_id..end_id) - - duplicate_tokens = Project.base_query - .where(runners_token: batch_records.select(:runners_token).distinct) - .group(:runners_token) - .having('COUNT(*) > 1') - .pluck(:runners_token) - - batch_records.where(runners_token: duplicate_tokens).update_all(runners_token: nil) if duplicate_tokens.any? - - mark_job_as_succeeded(start_id, end_id) - end - - private - - def mark_job_as_succeeded(*arguments) - ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb b/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb deleted file mode 100644 index b61f2ee7f4c..00000000000 --- a/lib/gitlab/background_migration/update_timelogs_null_spent_at.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class to populate spent_at for timelogs - class UpdateTimelogsNullSpentAt - include Gitlab::Database::DynamicModelHelpers - - BATCH_SIZE = 100 - - def perform(start_id, stop_id) - define_batchable_model('timelogs', connection: connection) - .where(spent_at: nil, id: start_id..stop_id) - .each_batch(of: 100) do |subbatch| - batch_start, batch_end = subbatch.pick('min(id), max(id)') - - update_timelogs(batch_start, batch_end) - end - end - - def update_timelogs(batch_start, batch_stop) - execute(<<~SQL) - UPDATE timelogs - SET spent_at = created_at - WHERE spent_at IS NULL - AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop}; - SQL - end - - def connection - @connection ||= ApplicationRecord.connection - end - - def execute(sql) - connection.execute(sql) - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 592e75b1430..e785ce558db 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -10,6 +10,8 @@ module Gitlab attr_reader :project, :client, :errors, :users + ALREADY_IMPORTED_CACHE_KEY = 'bitbucket_cloud-importer/already-imported/%{project}/%{collection}' + def initialize(project) @project = project @client = Bitbucket::Client.new(project.import_data.credentials) @@ -31,6 +33,18 @@ module Gitlab private + def already_imported?(collection, iid) + Gitlab::Cache::Import::Caching.set_includes?(cache_key(collection), iid) + end + + def mark_as_imported(collection, iid) + Gitlab::Cache::Import::Caching.set_add(cache_key(collection), iid) + end + + def cache_key(collection) + format(ALREADY_IMPORTED_CACHE_KEY, project: project.id, collection: collection) + end + def handle_errors return unless errors.any? @@ -97,6 +111,8 @@ module Gitlab issue_type_id = ::WorkItems::Type.default_issue_type.id client.issues(repo).each_with_index do |issue, index| + next if already_imported?(:issues, issue.iid) + # If a user creates an issue while the import is in progress, this can lead to an import failure. # The workaround is to allocate IIDs before starting the importer. allocate_issues_internal_id!(project, client) if index == 0 @@ -127,6 +143,8 @@ module Gitlab updated_at: issue.updated_at ) + mark_as_imported(:issues, issue.iid) + metrics.issues_counter.increment gitlab_issue.labels << @labels[label_name] @@ -179,6 +197,8 @@ module Gitlab pull_requests = client.pull_requests(repo) pull_requests.each do |pull_request| + next if already_imported?(:pull_requests, pull_request.iid) + import_pull_request(pull_request) end end @@ -209,6 +229,8 @@ module Gitlab updated_at: pull_request.updated_at ) + mark_as_imported(:pull_requests, pull_request.iid) + metrics.merge_requests_counter.increment import_pull_request_comments(pull_request, merge_request) if merge_request.persisted? diff --git a/lib/gitlab/bitbucket_server_import/importer.rb b/lib/gitlab/bitbucket_server_import/importer.rb index 6b163cd1b2d..f3253027d57 100644 --- a/lib/gitlab/bitbucket_server_import/importer.rb +++ b/lib/gitlab/bitbucket_server_import/importer.rb @@ -442,10 +442,9 @@ module Gitlab end def uid(rep_object) - # We want this explicit to only be username on the FF - # Otherwise, match email. - # There should be no default fall-through on username. Fall-through to import user - if Feature.enabled?(:bitbucket_server_user_mapping_by_username) + # We want this to only match either username or email depending on the flag state. + # There should be no fall-through. + if Feature.enabled?(:bitbucket_server_user_mapping_by_username, type: :ops) find_user_id(by: :username, value: rep_object.author_username) else find_user_id(by: :email, value: rep_object.author_email) diff --git a/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb index cd09ac40e9f..e7a9adf2beb 100644 --- a/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb @@ -17,6 +17,8 @@ module Gitlab project.repository.import_repository(project.import_url) project.repository.fetch_as_mirror(project.import_url, refmap: refmap) + validate_repository_size! + update_clone_time end @@ -48,7 +50,13 @@ module Gitlab def update_clone_time project.touch(:last_repository_updated_at) end + + def validate_repository_size! + # Defined in EE + end end end end end + +Gitlab::BitbucketServerImport::Importers::RepositoryImporter.prepend_mod diff --git a/lib/gitlab/bitbucket_server_import/user_finder.rb b/lib/gitlab/bitbucket_server_import/user_finder.rb index f96454eb2cc..68bd2d4851a 100644 --- a/lib/gitlab/bitbucket_server_import/user_finder.rb +++ b/lib/gitlab/bitbucket_server_import/user_finder.rb @@ -24,7 +24,7 @@ module Gitlab def uid(object) # We want this to only match either username or email depending on the flag state. # There should be no fall-through. - if Feature.enabled?(:bitbucket_server_user_mapping_by_username) + if Feature.enabled?(:bitbucket_server_user_mapping_by_username, type: :ops) find_user_id(by: :username, value: object.is_a?(Hash) ? object[:author_username] : object.author_username) else find_user_id(by: :email, value: object.is_a?(Hash) ? object[:author_email] : object.author_email) diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 7fec6584ba3..8f2df29c320 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -162,13 +162,13 @@ module Gitlab def self.write_multiple(mapping, key_prefix: nil, timeout: TIMEOUT) with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |multi| + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| mapping.each do |raw_key, value| key = cache_key_for("#{key_prefix}#{raw_key}") validate_redis_value!(value) - multi.set(key, value, ex: timeout) + pipeline.set(key, value, ex: timeout) end end end diff --git a/lib/gitlab/cache/json_cache.rb b/lib/gitlab/cache/json_cache.rb new file mode 100644 index 00000000000..7450c7e540b --- /dev/null +++ b/lib/gitlab/cache/json_cache.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + class JsonCache + STRATEGY_KEY_COMPONENTS = { + revision: Gitlab.revision, + version: [Gitlab::VERSION, Rails.version] + }.freeze + + def initialize(options = {}) + @backend = options.fetch(:backend, Rails.cache) + @namespace = options.fetch(:namespace, nil) + @cache_key_strategy = options.fetch(:cache_key_strategy, :revision) + end + + def active? + if backend.respond_to?(:active?) + backend.active? + else + true + end + end + + def expire(key) + backend.delete(cache_key(key)) + end + + def read(key, klass = nil) + value = read_raw(key) + value = parse_value(value, klass) unless value.nil? + value + end + + def write(key, value, options = nil) + write_raw(key, value, options) + end + + def fetch(key, options = {}) + klass = options.delete(:as) + value = read(key, klass) + + return value unless value.nil? + + value = yield + + write(key, value, options) + + value + end + + private + + attr_reader :backend, :namespace, :cache_key_strategy + + def cache_key(key) + expanded_cache_key(key).compact.join(':').freeze + end + + def write_raw(_key, _value, _options) + raise NoMethodError + end + + def expanded_cache_key(_key) + raise NoMethodError + end + + def read_raw(_key) + raise NoMethodError + end + + def parse_value(value, klass) + case value + when Hash then parse_entry(value, klass) + when Array then parse_entries(value, klass) + else + value + end + end + + def parse_entry(raw, klass) + return unless valid_entry?(raw, klass) + return klass.new(raw) unless klass.ancestors.include?(ActiveRecord::Base) + + # When the cached value is a persisted instance of ActiveRecord::Base in + # some cases a relation can return an empty collection because scope.none! + # is being applied on ActiveRecord::Associations::CollectionAssociation#scope + # when the new_record? method incorrectly returns false. + # + # See https://gitlab.com/gitlab-org/gitlab/issues/9903#note_145329964 + klass.allocate.init_with(encode_for(klass, raw)) + end + + def encode_for(klass, raw) + # We have models that leave out some fields from the JSON export for + # security reasons, e.g. models that include the CacheMarkdownField. + # The ActiveRecord::AttributeSet we build from raw does know about + # these columns so we need manually set them. + missing_attributes = (klass.columns.map(&:name) - raw.keys) + missing_attributes.each { |column| raw[column] = nil } + + coder = {} + klass.new(raw).encode_with(coder) + coder["new_record"] = new_record?(raw, klass) + coder + end + + def new_record?(raw, klass) + raw.fetch(klass.primary_key, nil).blank? + end + + def valid_entry?(raw, klass) + return false unless klass && raw.is_a?(Hash) + + (raw.keys - klass.attribute_names).empty? + end + + def parse_entries(values, klass) + values.filter_map { |value| parse_entry(value, klass) } + end + end + end +end diff --git a/lib/gitlab/cache/json_caches/json_keyed.rb b/lib/gitlab/cache/json_caches/json_keyed.rb new file mode 100644 index 00000000000..701a49c23de --- /dev/null +++ b/lib/gitlab/cache/json_caches/json_keyed.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + module JsonCaches + class JsonKeyed < JsonCache + private + + def expanded_cache_key(key) + [namespace, key] + end + + def write_raw(key, value, options = nil) + raw_value = {} + + begin + read_value = backend.read(cache_key(key)) + read_value = Gitlab::Json.parse(read_value.to_s) unless read_value.nil? + raw_value = read_value if read_value.is_a?(Hash) + rescue JSON::ParserError + end + + raw_value[strategy_key_component] = value + backend.write(cache_key(key), raw_value.to_json, options) + end + + def read_raw(key) + value = backend.read(cache_key(key)) + value = Gitlab::Json.parse(value.to_s) unless value.nil? + value[strategy_key_component] if value.is_a?(Hash) + rescue JSON::ParserError + nil + end + + def strategy_key_component + Array.wrap(STRATEGY_KEY_COMPONENTS.fetch(cache_key_strategy)).compact.join(':').freeze + end + end + end + end +end diff --git a/lib/gitlab/cache/json_caches/redis_keyed.rb b/lib/gitlab/cache/json_caches/redis_keyed.rb new file mode 100644 index 00000000000..92709adef63 --- /dev/null +++ b/lib/gitlab/cache/json_caches/redis_keyed.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + module JsonCaches + class RedisKeyed < JsonCache + private + + def expanded_cache_key(key) + [namespace, key, *strategy_key_component] + end + + def write_raw(key, value, options) + backend.write(cache_key(key), value.to_json, options) + end + + def read_raw(key) + value = backend.read(cache_key(key)) + value = Gitlab::Json.parse(value.to_s) unless value.nil? + value + rescue JSON::ParserError + nil + end + + def strategy_key_component + STRATEGY_KEY_COMPONENTS.fetch(cache_key_strategy) + end + end + end + end +end diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb index fa7c4972c91..8be1e1716ec 100644 --- a/lib/gitlab/checks/branch_check.rb +++ b/lib/gitlab/checks/branch_check.rb @@ -13,7 +13,8 @@ module Gitlab create_protected_branch: 'You are not allowed to create protected branches on this project.', invalid_commit_create_protected_branch: 'You can only use an existing protected branch ref as the basis of a new protected branch.', non_web_create_protected_branch: 'You can only create protected branches using the web interface and API.', - prohibited_hex_branch_name: 'You cannot create a branch with a 40-character hexadecimal branch name.' + prohibited_hex_branch_name: 'You cannot create a branch with a 40-character hexadecimal branch name.', + invalid_branch_name: 'You cannot create a branch with an invalid name.' }.freeze LOG_MESSAGES = { @@ -45,6 +46,10 @@ module Gitlab if branch_name =~ %r{\A\h{40}(/|\z)} raise GitAccess::ForbiddenError, ERROR_MESSAGES[:prohibited_hex_branch_name] end + + unless Gitlab::GitRefValidator.validate(branch_name) + raise GitAccess::ForbiddenError, ERROR_MESSAGES[:invalid_branch_name] + end end def protected_branch_checks diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index 083c2448a0a..1186b532baf 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -18,7 +18,10 @@ module Gitlab return unless should_run_validations? return if commits.empty? - paths = project.repository.find_changed_paths(commits.map(&:sha)) + paths = project.repository.find_changed_paths( + commits.map(&:sha), merge_commit_diff_mode: :all_parents + ) + paths.each do |path| validate_path(path) end diff --git a/lib/gitlab/ci/badge/release/latest_release.rb b/lib/gitlab/ci/badge/release/latest_release.rb index 8d84a54787b..8f247006f1a 100644 --- a/lib/gitlab/ci/badge/release/latest_release.rb +++ b/lib/gitlab/ci/badge/release/latest_release.rb @@ -19,7 +19,8 @@ module Gitlab::Ci @release = ::ReleasesFinder.new( project, current_user, - order_by: opts[:order_by]).execute.first + order_by_for_latest: opts[:order_by], + latest: true).execute.first end def entity diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index bc7aad1b186..17b9f30db33 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -6,14 +6,15 @@ module Gitlab class Rules include ::Gitlab::Utils::StrongMemoize - Result = Struct.new(:when, :start_in, :allow_failure, :variables, :needs, :errors) do + Result = Struct.new(:when, :start_in, :allow_failure, :variables, :needs, :errors, keyword_init: true) do def build_attributes + needs_job = needs&.dig(:job) { when: self.when, options: { start_in: start_in }.compact, allow_failure: allow_failure, - scheduling_type: (:dag if needs), - needs_attributes: needs&.[](:job) + scheduling_type: (:dag if needs_job.present?), + needs_attributes: needs_job }.compact end @@ -29,20 +30,25 @@ module Gitlab def evaluate(pipeline, context) if @rule_list.nil? - Result.new(@default_when) + Result.new(when: @default_when) elsif matched_rule = match_rule(pipeline, context) - Result.new( - matched_rule.attributes[:when] || @default_when, - matched_rule.attributes[:start_in], - matched_rule.attributes[:allow_failure], - matched_rule.attributes[:variables], - (matched_rule.attributes[:needs] if Feature.enabled?(:introduce_rules_with_needs, pipeline.project)) + result = Result.new( + when: matched_rule.attributes[:when] || @default_when, + start_in: matched_rule.attributes[:start_in], + allow_failure: matched_rule.attributes[:allow_failure], + variables: matched_rule.attributes[:variables] ) + + if Feature.enabled?(:introduce_rules_with_needs, pipeline.project) + result.needs = matched_rule.attributes[:needs] + end + + result else - Result.new('never') + Result.new(when: 'never') end rescue Rule::Clause::ParseError => e - Result.new('never', nil, nil, nil, nil, [e.message]) + Result.new(when: 'never', errors: [e.message]) end private diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index b3ff74c14da..b3862b3f186 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -10,7 +10,7 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable ALLOWED_KEYS = %i[key untracked paths when policy unprotect fallback_keys].freeze - ALLOWED_POLICY = %w[pull-push push pull].freeze + ALLOWED_POLICY = /pull-push|push|pull|\$\w{1,255}*/ DEFAULT_POLICY = 'pull-push' ALLOWED_WHEN = %w[on_success on_failure always].freeze DEFAULT_WHEN = 'on_success' @@ -18,9 +18,9 @@ module Gitlab validations do validates :config, type: Hash, allowed_keys: ALLOWED_KEYS - validates :policy, type: String, allow_blank: true, inclusion: { - in: ALLOWED_POLICY, - message: "should be one of: #{ALLOWED_POLICY.join(', ')}" + validates :policy, type: String, allow_blank: true, format: { + with: ALLOWED_POLICY, + message: "should be a variable or one of: pull-push, push, pull" } with_options allow_nil: true do diff --git a/lib/gitlab/ci/config/entry/include/rules/rule.rb b/lib/gitlab/ci/config/entry/include/rules/rule.rb index fa99a7204d6..9cdbd8cd037 100644 --- a/lib/gitlab/ci/config/entry/include/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/include/rules/rule.rb @@ -9,10 +9,14 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[if exists].freeze + ALLOWED_KEYS = %i[if exists when].freeze + ALLOWED_WHEN = %w[never always].freeze - attributes :if, :exists + attributes :if, :exists, :when + # Include rules are validated before Entry validations. This is because + # the include files are expanded before `compose!` runs in Ci::Config. + # The actual validation logic is in lib/gitlab/ci/config/external/rules.rb. validations do validates :config, presence: true validates :config, type: { with: Hash } diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 6b635cdf33b..61d95c8d4e6 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -130,8 +130,7 @@ module Gitlab strong_memoize_attr :content_hash def interpolator - External::Interpolator - .new(content_result, content_inputs, context) + Yaml::Interpolator.new(content_result, content_inputs, context) end strong_memoize_attr :interpolator diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index 16a6bc8a692..de726b57053 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -68,8 +68,6 @@ module Gitlab private def project - return legacy_project if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) - # Although we use `where_full_path_in`, this BatchLoader does not reduce the number of queries to 1. # That's because we use it in the `can_access_local_content?` and `sha` BatchLoaders # as the `for` parameter. And this loads the project immediately. @@ -83,10 +81,6 @@ module Gitlab end def can_access_local_content? - if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) - return legacy_can_access_local_content? - end - return if project.nil? # We are force-loading the project with the `itself` method @@ -103,7 +97,6 @@ module Gitlab end def sha - return legacy_sha if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) return if project.nil? # with `itself`, we are force-loading the project @@ -128,26 +121,6 @@ module Gitlab end end - def legacy_project - strong_memoize(:legacy_project) do - ::Project.find_by_full_path(project_name) - end - end - - def legacy_can_access_local_content? - strong_memoize(:legacy_can_access_local_content) do - context.logger.instrument(:config_file_project_validate_access) do - Ability.allowed?(context.user, :download_code, project) - end - end - end - - def legacy_sha - strong_memoize(:legacy_sha) do - project.commit(ref_name).try(:sha) - end - end - override :expand_context_attrs def expand_context_attrs { diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb index 3472f2c581a..95975e4661b 100644 --- a/lib/gitlab/ci/config/external/mapper/verifier.rb +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -11,10 +11,6 @@ module Gitlab # rubocop: disable Metrics/CyclomaticComplexity def process_without_instrumentation(files) - if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) - return legacy_process_without_instrumentation(files) - end - files.each do |file| # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an # "internal" `include`. We use this condition to exclude that `include` from the included file set. @@ -45,30 +41,6 @@ module Gitlab end # rubocop: enable Metrics/CyclomaticComplexity - def legacy_process_without_instrumentation(files) - files.each do |file| - # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an - # "internal" `include`. We use this condition to exclude that `include` from the included file set. - context.expandset << file unless context.internal_include? - verify_max_includes! - - verify_execution_time! - - file.validate_location! - file.validate_context! if file.valid? - file.content if file.valid? - end - - # We do not combine the loops because we need to load the content of all files before continuing - # to call `BatchLoader` for all locations. - files.each do |file| # rubocop:disable Style/CombinableLoops - verify_execution_time! - - file.validate_content! if file.valid? - file.load_and_validate_expanded_hash! if file.valid? - end - end - def verify_max_includes! return if context.expandset.count <= context.max_includes diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb index 95470537de3..134306332e6 100644 --- a/lib/gitlab/ci/config/external/rules.rb +++ b/lib/gitlab/ci/config/external/rules.rb @@ -6,6 +6,7 @@ module Gitlab module External class Rules ALLOWED_KEYS = Entry::Include::Rules::Rule::ALLOWED_KEYS + ALLOWED_WHEN = Entry::Include::Rules::Rule::ALLOWED_WHEN InvalidIncludeRulesError = Class.new(Mapper::Error) @@ -16,7 +17,17 @@ module Gitlab end def evaluate(context) - Result.new(@rule_list.nil? || match_rule(context)) + if Feature.enabled?(:ci_support_include_rules_when_never, context.project) + if @rule_list.nil? + Result.new('always') + elsif matched_rule = match_rule(context) + Result.new(matched_rule.attributes[:when]) + else + Result.new('never') + end + else + LegacyResult.new(@rule_list.nil? || match_rule(context)) + end end private @@ -29,13 +40,23 @@ module Gitlab return unless rule_hashes.is_a?(Array) rule_hashes.each do |rule_hash| - next if (rule_hash.keys - ALLOWED_KEYS).empty? + next if (rule_hash.keys - ALLOWED_KEYS).empty? && valid_when?(rule_hash) raise InvalidIncludeRulesError, "invalid include rule: #{rule_hash}" end end - Result = Struct.new(:result) do + def valid_when?(rule_hash) + rule_hash[:when].nil? || rule_hash[:when].in?(ALLOWED_WHEN) + end + + Result = Struct.new(:when) do + def pass? + self.when != 'never' + end + end + + LegacyResult = Struct.new(:result) do def pass? !!result end diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index 729e7e3ac05..f74ef95a832 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -4,51 +4,6 @@ module Gitlab module Ci class Config module Yaml - AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze - MAX_DOCUMENTS = 2 - - class Loader - def initialize(content, project: nil) - @content = content - @project = project - end - - def load! - ensure_custom_tags - - if project.present? && ::Feature.enabled?(:ci_multi_doc_yaml, project) - ::Gitlab::Config::Loader::MultiDocYaml.new( - content, - max_documents: MAX_DOCUMENTS, - additional_permitted_classes: AVAILABLE_TAGS, - reject_empty: true - ).load! - else - ::Gitlab::Config::Loader::Yaml - .new(content, additional_permitted_classes: AVAILABLE_TAGS) - .load! - end - end - - def to_result - Yaml::Result.new(config: load!, error: nil) - rescue ::Gitlab::Config::Loader::FormatError => e - Yaml::Result.new(error: e) - end - - private - - attr_reader :content, :project - - def ensure_custom_tags - @ensure_custom_tags ||= begin - AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } - - true - end - end - end - class << self def load!(content, project: nil) Loader.new(content, project: project).to_result.then do |result| diff --git a/lib/gitlab/ci/config/external/interpolator.rb b/lib/gitlab/ci/config/yaml/interpolator.rb index f8af77fb246..4ae191dfedf 100644 --- a/lib/gitlab/ci/config/external/interpolator.rb +++ b/lib/gitlab/ci/config/yaml/interpolator.rb @@ -3,9 +3,9 @@ module Gitlab module Ci class Config - module External + module Yaml ## - # Config::External::Interpolation perform includable file interpolation, and surfaces all possible interpolation + # Config::Yaml::Interpolation performs includable file interpolation, and surfaces all possible interpolation # errors. It is designed to provide an external file's validation context too. # class Interpolator diff --git a/lib/gitlab/ci/config/yaml/loader.rb b/lib/gitlab/ci/config/yaml/loader.rb new file mode 100644 index 00000000000..924a1f2e46b --- /dev/null +++ b/lib/gitlab/ci/config/yaml/loader.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + class Loader + AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze + MAX_DOCUMENTS = 2 + + def initialize(content, project: nil) + @content = content + @project = project + end + + def to_result + Yaml::Result.new(config: load!, error: nil) + rescue ::Gitlab::Config::Loader::FormatError => e + Yaml::Result.new(error: e) + end + + private + + attr_reader :content, :project + + def ensure_custom_tags + @ensure_custom_tags ||= begin + AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } + + true + end + end + + def load! + ensure_custom_tags + + ::Gitlab::Config::Loader::MultiDocYaml.new( + content, + max_documents: MAX_DOCUMENTS, + additional_permitted_classes: AVAILABLE_TAGS, + reject_empty: true + ).load! + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb index 33f9a454106..6b53adc3a57 100644 --- a/lib/gitlab/ci/config/yaml/result.rb +++ b/lib/gitlab/ci/config/yaml/result.rb @@ -31,7 +31,7 @@ module Gitlab def content return @config.last if has_header? - @config.first + @config.first || {} end end end diff --git a/lib/gitlab/ci/decompressed_gzip_size_validator.rb b/lib/gitlab/ci/decompressed_gzip_size_validator.rb index a92f3007671..9b7b5f0dd66 100644 --- a/lib/gitlab/ci/decompressed_gzip_size_validator.rb +++ b/lib/gitlab/ci/decompressed_gzip_size_validator.rb @@ -63,7 +63,7 @@ module Gitlab end def validate_archive_path - Gitlab::Utils.check_path_traversal!(archive_path) + Gitlab::PathTraversal.check_path_traversal!(archive_path) raise(ServiceError, 'Archive path is a symlink') if File.lstat(archive_path).symlink? raise(ServiceError, 'Archive path is not a file') unless File.file?(archive_path) diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index aff30455d09..9e71a9e8e91 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -42,11 +42,36 @@ module Gitlab end def custom_claims - super.merge( + additional_claims = { runner_id: runner&.id, runner_environment: runner_environment, sha: pipeline.sha + } + + if Feature.enabled?(:ci_jwt_v2_ref_uri_claim, pipeline.project) + additional_claims[:ci_config_ref_uri] = ci_config_ref_uri + end + + super.merge(additional_claims) + end + + def ci_config_ref_uri + project_config = Gitlab::Ci::ProjectConfig.new( + project: project, + sha: pipeline.sha, + pipeline_source: pipeline.source&.to_sym, + pipeline_source_bridge: pipeline.source_bridge ) + + return unless project_config&.source == :repository_source + + "#{project_config.url}@#{pipeline.source_ref_path}" + + # Errors are rescued to mitigate risk. This can be removed if no errors are observed. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117923#note_1387660746 for context. + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, pipeline_id: pipeline.id) + nil end def runner_environment diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 447136df81f..21408beb8cb 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -279,7 +279,6 @@ module Gitlab end def finding_name(data, identifiers, location) - return data['message'] if data['message'].present? return data['name'] if data['name'].present? identifier = identifiers.find(&:cve?) || identifiers.find(&:cwe?) || identifiers.first diff --git a/lib/gitlab/ci/project_config.rb b/lib/gitlab/ci/project_config.rb index 00b2ad58428..ffff2da9d7a 100644 --- a/lib/gitlab/ci/project_config.rb +++ b/lib/gitlab/ci/project_config.rb @@ -25,7 +25,7 @@ module Gitlab @config = find_config(project, sha, custom_content, pipeline_source, pipeline_source_bridge) end - delegate :content, :source, to: :@config, allow_nil: true + delegate :content, :source, :url, to: :@config, allow_nil: true delegate :internal_include_prepended?, to: :@config def exists? diff --git a/lib/gitlab/ci/project_config/repository.rb b/lib/gitlab/ci/project_config/repository.rb index 272425fd546..7dfd528fd6f 100644 --- a/lib/gitlab/ci/project_config/repository.rb +++ b/lib/gitlab/ci/project_config/repository.rb @@ -20,6 +20,11 @@ module Gitlab :repository_source end + override :url + def url + File.join(Settings.build_ci_component_fqdn, project.full_path, '//', ci_config_path) + end + private def file_in_repository? diff --git a/lib/gitlab/ci/project_config/source.rb b/lib/gitlab/ci/project_config/source.rb index 9a4a6394fa1..68853ca8296 100644 --- a/lib/gitlab/ci/project_config/source.rb +++ b/lib/gitlab/ci/project_config/source.rb @@ -5,6 +5,7 @@ module Gitlab class ProjectConfig class Source include Gitlab::Utils::StrongMemoize + extend ::Gitlab::Utils::Override def initialize(project, sha, custom_content, pipeline_source, pipeline_source_bridge) @project = project @@ -33,6 +34,10 @@ module Gitlab raise NotImplementedError end + def url + nil + end + private attr_reader :project, :sha, :custom_content, :pipeline_source, :pipeline_source_bridge diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index bf48c7d0bb7..d439149158a 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -82,7 +82,6 @@ module Gitlab details signatures description - message cve solution ].index_with do |key| @@ -174,10 +173,6 @@ module Gitlab original_data['description'] end - def message - original_data['message'] - end - def solution original_data['solution'] end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index 1bf015a0aa0..3040acc1eb4 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -112,3 +112,5 @@ module Gitlab end end end + +::Gitlab::Ci::RunnerInstructions.prepend_mod diff --git a/lib/gitlab/ci/secure_files/migration_helper.rb b/lib/gitlab/ci/secure_files/migration_helper.rb new file mode 100644 index 00000000000..13796f2476e --- /dev/null +++ b/lib/gitlab/ci/secure_files/migration_helper.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module SecureFiles + class MigrationHelper + class << self + def migrate_to_remote_storage(&block) + migrate_in_batches( + ::Ci::SecureFile.with_files_stored_locally, + ::Ci::SecureFileUploader::Store::REMOTE, + &block + ) + end + + private + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i + end + + def migrate_in_batches(files, store, &block) + files.find_each(batch_size: batch_size) do |file| # rubocop:disable CodeReuse/ActiveRecord + file.file.migrate!(store) + + yield file if block + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/waiting_for_approval.rb b/lib/gitlab/ci/status/build/waiting_for_approval.rb index ac3f5838d26..c5fb32034ce 100644 --- a/lib/gitlab/ci/status/build/waiting_for_approval.rb +++ b/lib/gitlab/ci/status/build/waiting_for_approval.rb @@ -5,44 +5,14 @@ module Gitlab module Status module Build class WaitingForApproval < Status::Extended - def illustration - { - image: 'illustrations/manual_action.svg', - size: 'svg-394', - title: _('Waiting for approval'), - content: _("This job deploys to the protected environment \"%{environment}\" which requires approvals.") % { environment: subject.deployment&.environment&.name } - } - end - - def has_action? - true - end - - def action_icon - nil - end - - def action_title - nil - end - - def action_button_title - _('Go to environments page to approve or reject') - end - - def action_path - project_environments_path(subject.project) - end - - def action_method - :get - end - + ## Extended in EE def self.matches?(build, user) - build.waiting_for_deployment_approval? + false end end end end end end + +Gitlab::Ci::Status::Build::WaitingForApproval.prepend_mod_with('Gitlab::Ci::Status::Build::WaitingForApproval') diff --git a/lib/gitlab/ci/status/scheduled.rb b/lib/gitlab/ci/status/scheduled.rb index e9068c326cf..8526becfef9 100644 --- a/lib/gitlab/ci/status/scheduled.rb +++ b/lib/gitlab/ci/status/scheduled.rb @@ -5,11 +5,11 @@ module Gitlab module Status class Scheduled < Status::Core def text - s_('CiStatusText|delayed') + s_('CiStatusText|scheduled') end def label - s_('CiStatusLabel|delayed') + s_('CiStatusLabel|scheduled') end def icon diff --git a/lib/gitlab/ci/status/success_warning.rb b/lib/gitlab/ci/status/success_warning.rb index 47623ad945f..84a0e52f518 100644 --- a/lib/gitlab/ci/status/success_warning.rb +++ b/lib/gitlab/ci/status/success_warning.rb @@ -9,7 +9,7 @@ module Gitlab # class SuccessWarning < Status::Extended def text - s_('CiStatusText|passed') + s_('CiStatusText|warning') end def label diff --git a/lib/gitlab/ci/templates/Android.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.gitlab-ci.yml index b8a4c59c233..95cdf9b9953 100644 --- a/lib/gitlab/ci/templates/Android.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android.gitlab-ci.yml @@ -6,53 +6,51 @@ # Read more about this script on this blog post https://about.gitlab.com/2018/10/24/setting-up-gitlab-ci-for-android-projects/, by Jason Lenny # If you are interested in using Android with FastLane for publishing take a look at the Android-Fastlane template. -image: openjdk:8-jdk +image: eclipse-temurin:17-jdk-jammy variables: # ANDROID_COMPILE_SDK is the version of Android you're compiling with. # It should match compileSdkVersion. - ANDROID_COMPILE_SDK: "29" + ANDROID_COMPILE_SDK: "33" # ANDROID_BUILD_TOOLS is the version of the Android build tools you are using. # It should match buildToolsVersion. - ANDROID_BUILD_TOOLS: "29.0.3" + ANDROID_BUILD_TOOLS: "33.0.2" # It's what version of the command line tools we're going to download from the official site. # Official Site-> https://developer.android.com/studio/index.html # There, look down below at the cli tools only, sdk tools package is of format: # commandlinetools-os_type-ANDROID_SDK_TOOLS_latest.zip # when the script was last modified for latest compileSdkVersion, it was which is written down below - ANDROID_SDK_TOOLS: "6514223" + ANDROID_SDK_TOOLS: "9477386" # Packages installation before running script before_script: - apt-get --quiet update --yes - - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 + - apt-get --quiet install --yes wget unzip # Setup path as android_home for moving/exporting the downloaded sdk into it - - export ANDROID_HOME="${PWD}/android-home" + - export ANDROID_HOME="${PWD}/android-sdk-root" # Create a new directory at specified location - install -d $ANDROID_HOME # Here we are installing androidSDK tools from official source, # (the key thing here is the url from where you are downloading these sdk tool for command line, so please do note this url pattern there and here as well) # after that unzipping those tools and # then running a series of SDK manager commands to install necessary android SDK packages that'll allow the app to build - - wget --output-document=$ANDROID_HOME/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip - # move to the archive at ANDROID_HOME - - pushd $ANDROID_HOME - - unzip -d cmdline-tools cmdline-tools.zip - - popd - - export PATH=$PATH:${ANDROID_HOME}/cmdline-tools/tools/bin/ + - wget --no-verbose --output-document=$ANDROID_HOME/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip + - unzip -q -d "$ANDROID_HOME/cmdline-tools" "$ANDROID_HOME/cmdline-tools.zip" + - mv -T "$ANDROID_HOME/cmdline-tools/cmdline-tools" "$ANDROID_HOME/cmdline-tools/tools" + - export PATH=$PATH:$ANDROID_HOME/cmdline-tools/latest/bin:$ANDROID_HOME/cmdline-tools/tools/bin # Nothing fancy here, just checking sdkManager version - sdkmanager --version # use yes to accept all licenses - - yes | sdkmanager --sdk_root=${ANDROID_HOME} --licenses || true - - sdkmanager --sdk_root=${ANDROID_HOME} "platforms;android-${ANDROID_COMPILE_SDK}" - - sdkmanager --sdk_root=${ANDROID_HOME} "platform-tools" - - sdkmanager --sdk_root=${ANDROID_HOME} "build-tools;${ANDROID_BUILD_TOOLS}" + - yes | sdkmanager --licenses > /dev/null || true + - sdkmanager "platforms;android-${ANDROID_COMPILE_SDK}" + - sdkmanager "platform-tools" + - sdkmanager "build-tools;${ANDROID_BUILD_TOOLS}" # Not necessary, but just for surity - chmod +x ./gradlew @@ -64,6 +62,11 @@ lintDebug: stage: build script: - ./gradlew -Pci --console=plain :app:lintDebug -PbuildDir=lint + artifacts: + paths: + - app/lint/reports/lint-results-debug.html + expose_as: "lint-report" + when: always # Make Project assembleDebug: @@ -77,6 +80,7 @@ assembleDebug: # Run all tests, if any fails, interrupt the pipeline(fail it) debugTests: + needs: [lintDebug, assembleDebug] interruptible: true stage: test script: diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml index 7f81755348c..3d053d4d78c 100644 --- a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -8,9 +8,9 @@ code_quality: stage: test - image: "cirrusci/flutter:1.22.5" + image: "ghcr.io/cirruslabs/flutter:3.10.3" before_script: - - pub global activate dart_code_metrics + - flutter pub global activate dart_code_metrics - export PATH="$PATH:$HOME/.pub-cache/bin" script: - metrics lib -r codeclimate > gl-code-quality-report.json @@ -20,9 +20,9 @@ code_quality: test: stage: test - image: "cirrusci/flutter:1.22.5" + image: "ghcr.io/cirruslabs/flutter:3.10.3" before_script: - - pub global activate junitreport + - flutter pub global activate junitreport - export PATH="$PATH:$HOME/.pub-cache/bin" script: - flutter test --machine --coverage | tojunit -o report.xml diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 7a4c65f8c5b..49d3c270bac 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.32.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.34.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 7a4c65f8c5b..49d3c270bac 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.32.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.34.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml index 6e8cf15204a..de3f688bdb6 100644 --- a/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/CF-Provision.gitlab-ci.yml @@ -7,7 +7,7 @@ cloud_formation: script: - gl-cloudformation create-stack rules: - - if: '($AUTO_DEVOPS_PLATFORM_TARGET != "EC2") || ($AUTO_DEVOPS_PLATFORM_TARGET != "ECS")' + - if: '($AUTO_DEVOPS_PLATFORM_TARGET != "EC2") && ($AUTO_DEVOPS_PLATFORM_TARGET != "ECS")' when: never - if: '$CI_KUBERNETES_ACTIVE || $KUBECONFIG' when: never diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index b2ab6704e35..7c9aa82b1ae 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE_TAG: "0.94.0" + CODE_QUALITY_IMAGE_TAG: "0.96.0" CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG" needs: [] script: diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 4ee5fa74df9..f4a13d61ba2 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.2' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.50.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 622b44d78ad..c1a3daa7f5b 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.2' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.50.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 2954ddf8a35..a3c7c6baf02 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.2' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.50.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml index d46ac97ad1b..d7a6104082d 100644 --- a/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml @@ -32,11 +32,11 @@ kaniko-build: VERSION="latest" elif [ -n "$CI_COMMIT_TAG" ];then NOSLASH=$(echo "$CI_COMMIT_TAG" | tr -s / - ) - SANITIZED="${NOSLASH//[^a-zA-Z0-9\-\.]/}" + SANITIZED="${NOSLASH//[^a-zA-Z0-9.-]/}" VERSION="$SANITIZED" else \ NOSLASH=$(echo "$CI_COMMIT_REF_NAME" | tr -s / - ) - SANITIZED="${NOSLASH//[^a-zA-Z0-9\-]/}" + SANITIZED="${NOSLASH//[^a-zA-Z0-9-]/}" VERSION="branch-$SANITIZED" fi export IMAGE_TAG=$CI_REGISTRY_IMAGE:$VERSION diff --git a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml index 51d2273d41d..eb1c920e11b 100644 --- a/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Brunch.gitlab-ci.yml -# Full project: https://gitlab.com/pages/brunch -image: node:4.2.2 +default: + # Full project: https://gitlab.com/pages/brunch + image: node:4.2.2 pages: cache: diff --git a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml index e577a489c55..95eab65e629 100644 --- a/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Doxygen.gitlab-ci.yml -# Full project: https://gitlab.com/pages/doxygen -image: alpine +default: + # Full project: https://gitlab.com/pages/doxygen + image: alpine pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index 88ed73b41e7..c58db066d61 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -3,13 +3,14 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml -image: node:latest +default: + image: node:latest -# This folder is cached between builds -# https://docs.gitlab.com/ee/ci/yaml/index.html#cache -cache: - paths: - - node_modules/ + # This folder is cached between builds + # https://docs.gitlab.com/ee/ci/yaml/index.html#cache + cache: + paths: + - node_modules/ pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml index aa86ad2a6ad..313cb99a841 100644 --- a/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Harp.gitlab-ci.yml -# Full project: https://gitlab.com/pages/harp -image: node:4.2.2 +default: + # Full project: https://gitlab.com/pages/harp + image: node:4.2.2 pages: cache: diff --git a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml index b1617e9239c..f27228f2e3c 100644 --- a/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Hexo.gitlab-ci.yml -# Full project: https://gitlab.com/pages/hexo -image: node:10.15.3 +default: + # Full project: https://gitlab.com/pages/hexo + image: node:10.15.3 pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml index d6f6e94526e..6a364959db2 100644 --- a/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hugo.gitlab-ci.yml @@ -6,7 +6,8 @@ --- # All available Hugo versions are listed here: # https://gitlab.com/pages/hugo/container_registry -image: "${CI_TEMPLATE_REGISTRY_HOST}/pages/hugo:latest" +default: + image: "${CI_TEMPLATE_REGISTRY_HOST}/pages/hugo:latest" variables: GIT_SUBMODULE_STRATEGY: recursive @@ -14,9 +15,8 @@ variables: test: script: - hugo - except: - variables: - - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + rules: + - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH pages: script: @@ -24,7 +24,6 @@ pages: artifacts: paths: - public - only: - variables: - - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH environment: production diff --git a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml index fba4afca9ed..c3ee4d62359 100644 --- a/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml @@ -3,12 +3,13 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Hyde.gitlab-ci.yml -# Full project: https://gitlab.com/pages/hyde -image: python:2.7 +default: + # Full project: https://gitlab.com/pages/hyde + image: python:2.7 -cache: - paths: - - vendor/ + cache: + paths: + - vendor/ test: stage: test diff --git a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml index 57e3ced4dc2..ba8eb81ca22 100644 --- a/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/JBake.gitlab-ci.yml @@ -13,20 +13,21 @@ # # HowTo at: https://jorge.aguilera.gitlab.io/howtojbake/ -image: java:8 +default: + image: java:8 + + # We use SDKMan as tool for managing versions + before_script: + - apt-get update -qq && apt-get install -y -qq unzip zip + - curl -sSL https://get.sdkman.io | bash + - echo sdkman_auto_answer=true > /root/.sdkman/etc/config + - source /root/.sdkman/bin/sdkman-init.sh + - sdk install jbake $JBAKE_VERSION < /dev/null + - sdk use jbake $JBAKE_VERSION variables: JBAKE_VERSION: 2.5.1 -# We use SDKMan as tool for managing versions -before_script: - - apt-get update -qq && apt-get install -y -qq unzip zip - - curl -sSL https://get.sdkman.io | bash - - echo sdkman_auto_answer=true > /root/.sdkman/etc/config - - source /root/.sdkman/bin/sdkman-init.sh - - sdk install jbake $JBAKE_VERSION < /dev/null - - sdk use jbake $JBAKE_VERSION - # This build job produced the output directory of your site pages: environment: production diff --git a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml index 8b07454af24..812a08c33fb 100644 --- a/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml @@ -3,18 +3,19 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Jekyll.gitlab-ci.yml -# Template project: https://gitlab.com/pages/jekyll -# Docs: https://docs.gitlab.com/ee/pages/ -image: ruby:2.6 +default: + # Template project: https://gitlab.com/pages/jekyll + # Docs: https://docs.gitlab.com/ee/pages/ + image: ruby:2.6 + + before_script: + - gem install bundler + - bundle install variables: JEKYLL_ENV: production LC_ALL: C.UTF-8 -before_script: - - gem install bundler - - bundle install - test: stage: test script: diff --git a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml index ad083fcc5db..291c03c0002 100644 --- a/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Jigsaw.gitlab-ci.yml @@ -7,29 +7,30 @@ # # Full project: https://github.com/tightenco/jigsaw -image: php:7.2 +default: + image: php:7.2 -# These folders are cached between builds -cache: - paths: - - vendor/ - - node_modules/ + # These folders are cached between builds + cache: + paths: + - vendor/ + - node_modules/ -before_script: - # Update packages - - apt-get update -yqq - # Install dependencies - - apt-get install -yqq gnupg zlib1g-dev libpng-dev - # Install Node 8 - - curl -sL https://deb.nodesource.com/setup_8.x | bash - - - apt-get install -yqq nodejs - # Install php extensions - - docker-php-ext-install zip - # Install Composer and project dependencies - - curl -sS https://getcomposer.org/installer | php - - php composer.phar install - # Install Node dependencies - - npm install + before_script: + # Update packages + - apt-get update -yqq + # Install dependencies + - apt-get install -yqq gnupg zlib1g-dev libpng-dev + # Install Node 8 + - curl -sL https://deb.nodesource.com/setup_8.x | bash - + - apt-get install -yqq nodejs + # Install php extensions + - docker-php-ext-install zip + # Install Composer and project dependencies + - curl -sS https://getcomposer.org/installer | php + - php composer.phar install + # Install Node dependencies + - npm install pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml index e86337ae23c..e83cf30e999 100644 --- a/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Lektor.gitlab-ci.yml -# Full project: https://gitlab.com/pages/hyde -image: python:2.7 +default: + # Full project: https://gitlab.com/pages/hyde + image: python:2.7 pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml index a49e95b62c8..bbbffa7c682 100644 --- a/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Metalsmith.gitlab-ci.yml -# Full project: https://gitlab.com/pages/metalsmith -image: node:4.2.2 +default: + # Full project: https://gitlab.com/pages/metalsmith + image: node:4.2.2 pages: cache: diff --git a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml index d8f036ab4ed..f0a7f88eaf2 100644 --- a/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml @@ -3,12 +3,13 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Middleman.gitlab-ci.yml -# Full project: https://gitlab.com/pages/middleman -image: ruby:2.6 +default: + # Full project: https://gitlab.com/pages/middleman + image: ruby:2.6 -cache: - paths: - - vendor + cache: + paths: + - vendor test: script: diff --git a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml index b0511abd109..3160948ec35 100644 --- a/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Nanoc.gitlab-ci.yml -# Full project: https://gitlab.com/pages/nanoc -image: ruby:2.6 +default: + # Full project: https://gitlab.com/pages/nanoc + image: ruby:2.6 pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml index c89050eede7..395842c4cd1 100644 --- a/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Octopress.gitlab-ci.yml -# Full project: https://gitlab.com/pages/octopress -image: ruby:2.6 +default: + # Full project: https://gitlab.com/pages/octopress + image: ruby:2.6 pages: script: diff --git a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml index 3721344b21e..7921a71a89a 100644 --- a/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml @@ -3,8 +3,9 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Pelican.gitlab-ci.yml -# Full project: https://gitlab.com/pages/pelican -image: python:2.7-alpine +default: + # Full project: https://gitlab.com/pages/pelican + image: python:2.7-alpine pages: script: diff --git a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml index 00efcfa1b32..d75ecc3da01 100644 --- a/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml @@ -3,7 +3,13 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/SwaggerUI.gitlab-ci.yml -image: node:10-alpine +default: + image: node:10-alpine + + # These folders are cached between builds + cache: + paths: + - ./node_modules # specify the location of the Open API Specification files within your project # and the filename of the specification that you would like to display by default @@ -11,11 +17,6 @@ variables: DOCS_FOLDER: "api-docs" SPEC_TO_DISPLAY: "my-project_specification_0.0.1.json" -# These folders are cached between builds -cache: - paths: - - ./node_modules - # publishes all files from the $DOCS_FOLDER together with the static version of SwaggerUI # sets the specification file named in $SPEC_TO_DISPLAY to be displayed by default pages: diff --git a/lib/gitlab/ci/templates/Pages/Zola.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Zola.gitlab-ci.yml new file mode 100644 index 00000000000..c2c890846b9 --- /dev/null +++ b/lib/gitlab/ci/templates/Pages/Zola.gitlab-ci.yml @@ -0,0 +1,30 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Pages/Zola.gitlab-ci.yml + +# Prefer to copy-paste this template instead of include it to ensure forward compatibility + +--- +# From: https://www.getzola.org/documentation/deployment/gitlab-pages/ +# Source template is slightly modified to be self-contained + +pages: + image: alpine:latest + variables: + # This variable will ensure that the CI runner pulls in your theme from the submodule + GIT_SUBMODULE_STRATEGY: recursive + before_script: + # Install the zola package from the alpine community repositories + - apk add --update-cache --repository http://dl-cdn.alpinelinux.org/alpine/edge/community/ zola + script: + # Execute zola build + - zola build --base-url "$CI_PAGES_URL" + artifacts: + paths: + # Path of our artifacts + - public + # This config will only publish changes that are pushed on the default branch + rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + environment: production diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 792bd7f666b..f10011ab23b 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -26,11 +26,12 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" + DAST_IMAGE_SUFFIX: "" dast: stage: dast image: - name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION" + name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION$DAST_IMAGE_SUFFIX" variables: GIT_STRATEGY: none allow_failure: true @@ -57,6 +58,11 @@ dast: $REVIEW_DISABLED == '1' when: never - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" && + $GITLAB_FEATURES =~ /\bdast\b/ + variables: + DAST_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ after_script: # Remove any debug.log files because they might contain secrets. diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index d1d1c4d7e52..989f9caf601 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -26,11 +26,12 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" + DAST_IMAGE_SUFFIX: "" dast: stage: dast image: - name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION" + name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION$DAST_IMAGE_SUFFIX" variables: GIT_STRATEGY: none allow_failure: true @@ -59,6 +60,12 @@ dast: $REVIEW_DISABLED == '1' when: never + # Add the job to merge request pipelines if there's an open merge request. (FIPS) + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $CI_GITLAB_FIPS_MODE == "true" && + $GITLAB_FEATURES =~ /\bdast\b/ + variables: + DAST_IMAGE_SUFFIX: "-fips" # Add the job to merge request pipelines if there's an open merge request. - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $GITLAB_FEATURES =~ /\bdast\b/ @@ -67,6 +74,12 @@ dast: - if: $CI_OPEN_MERGE_REQUESTS when: never + # Add the job to branch pipelines. (FIPS) + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" && + $GITLAB_FEATURES =~ /\bdast\b/ + variables: + DAST_IMAGE_SUFFIX: "-fips" # Add the job to branch pipelines. - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 88fe55a44ab..793030d302a 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -9,18 +9,19 @@ # There is a more opinionated template which we suggest the users to abide, # which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml -image: - name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/stable:latest" +default: + image: + name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/stable:latest" + + cache: + key: "${TF_ROOT}" + paths: + - ${TF_ROOT}/.terraform/ variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project TF_STATE_NAME: default # The name of the state file used by the GitLab Managed Terraform state backend -cache: - key: "${TF_ROOT}" - paths: - - ${TF_ROOT}/.terraform/ - .terraform:fmt: &terraform_fmt stage: validate script: diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 86e54fdfcdf..cae3a966bc6 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -139,14 +139,6 @@ module Gitlab # Set environment name here so we can access it when evaluating the job's rules variables.append(key: 'CI_ENVIRONMENT_NAME', value: job.environment) if job.environment - - if Feature.disabled?(:ci_remove_legacy_predefined_variables, project) - # legacy variables - variables.append(key: 'CI_BUILD_NAME', value: job.name) - variables.append(key: 'CI_BUILD_STAGE', value: job.stage_name) - variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request - variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action? - end end end diff --git a/lib/gitlab/ci/variables/builder/pipeline.rb b/lib/gitlab/ci/variables/builder/pipeline.rb index 1e7a18d70b0..c3b0cb856ba 100644 --- a/lib/gitlab/ci/variables/builder/pipeline.rb +++ b/lib/gitlab/ci/variables/builder/pipeline.rb @@ -40,7 +40,7 @@ module Gitlab attr_reader :pipeline - def predefined_commit_variables # rubocop:disable Metrics/AbcSize - Remove this rubocop:disable when FF `ci_remove_legacy_predefined_variables` is removed. + def predefined_commit_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| next variables unless pipeline.sha.present? @@ -56,24 +56,10 @@ module Gitlab variables.append(key: 'CI_COMMIT_REF_PROTECTED', value: (!!pipeline.protected_ref?).to_s) variables.append(key: 'CI_COMMIT_TIMESTAMP', value: pipeline.git_commit_timestamp.to_s) variables.append(key: 'CI_COMMIT_AUTHOR', value: pipeline.git_author_full_text.to_s) - - if Feature.disabled?(:ci_remove_legacy_predefined_variables, pipeline.project) - variables.concat(legacy_predefined_commit_variables) - end end end strong_memoize_attr :predefined_commit_variables - def legacy_predefined_commit_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_BUILD_REF', value: pipeline.sha) - variables.append(key: 'CI_BUILD_BEFORE_SHA', value: pipeline.before_sha) - variables.append(key: 'CI_BUILD_REF_NAME', value: pipeline.source_ref) - variables.append(key: 'CI_BUILD_REF_SLUG', value: pipeline.source_ref_slug) - end - end - strong_memoize_attr :legacy_predefined_commit_variables - def predefined_commit_tag_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| git_tag = pipeline.project.repository.find_tag(pipeline.ref) @@ -82,21 +68,10 @@ module Gitlab variables.append(key: 'CI_COMMIT_TAG', value: pipeline.ref) variables.append(key: 'CI_COMMIT_TAG_MESSAGE', value: git_tag.message) - - if Feature.disabled?(:ci_remove_legacy_predefined_variables, pipeline.project) - variables.concat(legacy_predefined_commit_tag_variables) - end end end strong_memoize_attr :predefined_commit_tag_variables - def legacy_predefined_commit_tag_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - variables.append(key: 'CI_BUILD_TAG', value: pipeline.ref) - end - end - strong_memoize_attr :legacy_predefined_commit_tag_variables - def predefined_merge_request_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI_MERGE_REQUEST_EVENT_TYPE', value: pipeline.merge_request_event_type.to_s) diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb deleted file mode 100644 index 957faf797b5..00000000000 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Cluster - class PumaWorkerKillerInitializer - def self.start( - puma_options, - puma_per_worker_max_memory_mb: 1200, - puma_master_max_memory_mb: 950, - additional_puma_dev_max_memory_mb: 200) - - # We are replacing PWK with Watchdog by using backward compatible RssMemoryLimit monitor by default. - # https://gitlab.com/groups/gitlab-org/-/epics/9119 - return if Gitlab::Utils.to_boolean(ENV.fetch('GITLAB_MEMORY_WATCHDOG_ENABLED', true)) - - require 'puma_worker_killer' - - PumaWorkerKiller.config do |config| - worker_count = puma_options[:workers] || 1 - # The Puma Worker Killer checks the total memory used by the cluster, - # i.e. both primary and worker processes. - # https://github.com/schneems/puma_worker_killer/blob/v0.1.0/lib/puma_worker_killer/puma_memory.rb#L57 - # - # Additional memory is added when running in `development` - config.ram = puma_master_max_memory_mb + - (worker_count * puma_per_worker_max_memory_mb) + - (Rails.env.development? ? (1 + worker_count) * additional_puma_dev_max_memory_mb : 0) - - config.frequency = 20 # seconds - - # We just want to limit to a fixed maximum, unrelated to the total amount - # of available RAM. - config.percent_usage = 0.98 - - # Ideally we'll never hit the maximum amount of memory. Restart the workers - # regularly rather than rely on OOM behavior for periodic restarting. - config.rolling_restart_frequency = 43200 # 12 hours in seconds. - - # Spread the rolling restarts out over 1 hour to avoid too many simultaneous - # process startups. - config.rolling_restart_splay_seconds = 0.0..3600.0 # 0 to 1 hour in seconds. - - observer = Gitlab::Cluster::PumaWorkerKillerObserver.new - config.pre_term = observer.callback - end - - PumaWorkerKiller.start - end - end - end -end diff --git a/lib/gitlab/cluster/puma_worker_killer_observer.rb b/lib/gitlab/cluster/puma_worker_killer_observer.rb deleted file mode 100644 index f53051c32ff..00000000000 --- a/lib/gitlab/cluster/puma_worker_killer_observer.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Cluster - class PumaWorkerKillerObserver - def initialize - @counter = Gitlab::Metrics.counter(:puma_killer_terminations_total, 'Number of workers terminated by PumaWorkerKiller') - end - - # returns the Proc to be used as the observer callback block - def callback - method(:log_termination) - end - - private - - def log_termination(worker) - @counter.increment - end - end - end -end diff --git a/lib/gitlab/counters/buffered_counter.rb b/lib/gitlab/counters/buffered_counter.rb index 3e232c78e45..258ada864c8 100644 --- a/lib/gitlab/counters/buffered_counter.rb +++ b/lib/gitlab/counters/buffered_counter.rb @@ -78,11 +78,7 @@ module Gitlab def increment(increment) result = redis_state do |redis| - if Feature.enabled?(:project_statistics_bulk_increment, type: :development) - redis.eval(LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT, **increment_args(increment)).to_i - else - redis.incrby(key, increment.amount) - end + redis.eval(LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT, **increment_args(increment)).to_i end FlushCounterIncrementsWorker.perform_in(WORKER_DELAY, counter_record.class.name, counter_record.id, attribute) @@ -94,11 +90,7 @@ module Gitlab result = redis_state do |redis| redis.pipelined do |pipeline| increments.each do |increment| - if Feature.enabled?(:project_statistics_bulk_increment, type: :development) - pipeline.eval(LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT, **increment_args(increment)) - else - pipeline.incrby(key, increment.amount) - end + pipeline.eval(LUA_INCREMENT_WITH_DEDUPLICATION_SCRIPT, **increment_args(increment)) end end end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index ecb0cc20a64..2fa0b40df14 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -65,6 +65,7 @@ module Gitlab { id: pipeline.id, iid: pipeline.iid, + name: pipeline.name, ref: pipeline.source_ref, tag: pipeline.tag, sha: pipeline.sha, @@ -77,7 +78,8 @@ module Gitlab finished_at: pipeline.finished_at, duration: pipeline.duration, queued_duration: pipeline.queued_duration, - variables: pipeline.variables.map(&:hook_attrs) + variables: pipeline.variables.map(&:hook_attrs), + url: Gitlab::Routing.url_helpers.project_pipeline_url(pipeline.project, pipeline) } end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 4197c87f51f..da9ebf4ab0f 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -2,8 +2,6 @@ module Gitlab module Database - DATABASE_NAMES = %w[main ci main_clusterwide].freeze - MAIN_DATABASE_NAME = 'main' CI_DATABASE_NAME = 'ci' DEFAULT_POOL_HEADROOM = 10 @@ -56,67 +54,78 @@ module Gitlab MODE_SINGLE_DATABASE_CI_CONNECTION = "single-database-ci-connection" MODE_MULTIPLE_DATABASES = "multiple-databases" + def self.all_database_connection_files + Dir.glob(Rails.root.join("db/database_connections/*.yaml")) + end + + def self.all_gitlab_schema_files + Dir.glob(Rails.root.join("db/gitlab_schemas/*.yaml")) + end + + def self.all_database_connections + @all_database_connections ||= + all_database_connection_files + .map { |file| DatabaseConnectionInfo.load_file(file) } + .sort_by(&:order) + .index_by(&:name) + .with_indifferent_access.freeze + end + + def self.all_database_names + all_database_connections.keys.map(&:to_s) + end + + def self.all_gitlab_schemas + @all_gitlab_schemas ||= + all_gitlab_schema_files + .map { |file| GitlabSchemaInfo.load_file(file) } + .index_by(&:name) + .with_indifferent_access.freeze + end + def self.database_base_models - @database_base_models ||= { - # Note that we use ActiveRecord::Base here and not ApplicationRecord. - # This is deliberate, as we also use these classes to apply load - # balancing to, and the load balancer must be enabled for _all_ models - # that inherit from ActiveRecord::Base; not just our own models that - # inherit from ApplicationRecord. - main: ::ActiveRecord::Base, - main_clusterwide: ::MainClusterwide::ApplicationRecord.connection_class? ? ::MainClusterwide::ApplicationRecord : nil, - ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil - }.compact.with_indifferent_access.freeze + # Note that we use ActiveRecord::Base here and not ApplicationRecord. + # This is deliberate, as we also use these classes to apply load + # balancing to, and the load balancer must be enabled for _all_ models + # that inherit from ActiveRecord::Base; not just our own models that + # inherit from ApplicationRecord. + @database_base_models ||= + all_database_connections + .transform_values(&:connection_class) + .compact.with_indifferent_access.freeze end # This returns a list of databases that contains all the gitlab_shared schema - # tables. We can't reuse database_base_models because Geo does not support - # the gitlab_shared tables yet. + # tables. def self.database_base_models_with_gitlab_shared - @database_base_models_with_gitlab_shared ||= { - # Note that we use ActiveRecord::Base here and not ApplicationRecord. - # This is deliberate, as we also use these classes to apply load - # balancing to, and the load balancer must be enabled for _all_ models - # that inher from ActiveRecord::Base; not just our own models that - # inherit from ApplicationRecord. - main: ::ActiveRecord::Base, - main_clusterwide: ::MainClusterwide::ApplicationRecord.connection_class? ? ::MainClusterwide::ApplicationRecord : nil, - ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil - }.compact.with_indifferent_access.freeze + @database_base_models_with_gitlab_shared ||= + all_database_connections + .select { |_, db| db.has_gitlab_shared? } + .transform_values(&:connection_class) + .compact.with_indifferent_access.freeze end # This returns a list of databases whose connection supports database load - # balancing. We can't reuse the database_base_models method because the Geo - # database does not support load balancing yet. - # - # TODO: https://gitlab.com/gitlab-org/geo-team/discussions/-/issues/5032 + # balancing. We can't reuse the database_base_models since not all connections + # do support load balancing. def self.database_base_models_using_load_balancing - @database_base_models_using_load_balancing ||= { - # Note that we use ActiveRecord::Base here and not ApplicationRecord. - # This is deliberate, as we also use these classes to apply load - # balancing to, and the load balancer must be enabled for _all_ models - # that inher from ActiveRecord::Base; not just our own models that - # inherit from ApplicationRecord. - main: ::ActiveRecord::Base, - main_clusterwide: ::MainClusterwide::ApplicationRecord.connection_class? ? ::MainClusterwide::ApplicationRecord : nil, - ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil - }.compact.with_indifferent_access.freeze + @database_base_models_using_load_balancing ||= + all_database_connections + .select { |_, db| db.uses_load_balancing? } + .transform_values(&:connection_class) + .compact.with_indifferent_access.freeze end # This returns a list of base models with connection associated for a given gitlab_schema def self.schemas_to_base_models - @schemas_to_base_models ||= { - gitlab_main: [self.database_base_models.fetch(:main)], - gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main - gitlab_shared: database_base_models_with_gitlab_shared.values, # all models - gitlab_internal: database_base_models.values, # all models - gitlab_pm: [self.database_base_models.fetch(:main)], # package metadata models - gitlab_main_clusterwide: [self.database_base_models[:main_clusterwide] || self.database_base_models.fetch(:main)] - }.with_indifferent_access.freeze - end - - def self.all_database_names - DATABASE_NAMES + @schemas_to_base_models ||= + all_gitlab_schemas.transform_values do |schema| + all_database_connections + .values + .select { |db| db.gitlab_schemas.include?(schema.name) } + .filter_map { |db| db.connection_class_or_fallback(all_database_connections) } + .uniq + end.compact.with_indifferent_access.freeze end # We configure the database connection pool size automatically based on the @@ -255,8 +264,16 @@ module Gitlab end end - def self.db_config_names - ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name) - ['geo'] + def self.db_config_names(with_schema:) + db_config_names = ::ActiveRecord::Base.configurations + .configs_for(env_name: Rails.env).map(&:name) + return db_config_names unless with_schema + + schema_models = schemas_to_base_models.fetch(with_schema) + db_config_names.select do |db_config_name| + db_info = all_database_connections.fetch(db_config_name) + schema_models.include?(db_info.connection_class) + end end # This returns all matching schemas that a given connection can use diff --git a/lib/gitlab/database/async_indexes/index_base.rb b/lib/gitlab/database/async_indexes/index_base.rb index bde75e12295..93f3ba88345 100644 --- a/lib/gitlab/database/async_indexes/index_base.rb +++ b/lib/gitlab/database/async_indexes/index_base.rb @@ -81,7 +81,8 @@ module Gitlab { table_name: async_index.table_name, index_name: async_index.name, - class: self.class.name.to_s + class: self.class.name.to_s, + connection_name: database_config_name } end end diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb index 9f5f39613ed..a3c600a4519 100644 --- a/lib/gitlab/database/async_indexes/postgres_async_index.rb +++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb @@ -8,13 +8,17 @@ module Gitlab self.table_name = 'postgres_async_indexes' + # schema_name + . + table_name + MAX_TABLE_NAME_LENGTH = (Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH * 2) + 1 MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH MAX_DEFINITION_LENGTH = 2048 validates :name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } - validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } + validates :table_name, presence: true, length: { maximum: MAX_TABLE_NAME_LENGTH } validates :definition, presence: true, length: { maximum: MAX_DEFINITION_LENGTH } + validate :ensure_correct_schema_and_table_name + scope :to_create, -> { where("definition ILIKE 'CREATE%'") } scope :to_drop, -> { where("definition ILIKE 'DROP%'") } scope :ordered, -> { order(attempts: :asc, id: :asc) } @@ -22,6 +26,24 @@ module Gitlab def to_s definition end + + private + + def ensure_correct_schema_and_table_name + return unless table_name + + schema, table, *rest = table_name.split('.') + + too_long = (table.nil? && schema.length > MAX_DEFINITION_LENGTH) || # no schema given + # both schema and table given + (schema.length > MAX_IDENTIFIER_LENGTH || (table && table.length > MAX_IDENTIFIER_LENGTH)) + + if too_long + errors.add(:table_name, :too_long) + elsif rest.any? + errors.add(:table_name, :invalid) + end + end end end end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 523ab2a9f27..458b099924b 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -139,7 +139,7 @@ module Gitlab new_batch_size = batch_size / 2 - break update!(attempts: 0) if new_batch_size < 1 + next update!(attempts: 0) if new_batch_size < 1 batching_strategy = batched_migration.batch_class.new(connection: self.class.connection) next_batch_bounds = batching_strategy.next_batch( diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index a883996a5c5..83beee091f1 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -79,6 +79,10 @@ module Gitlab transition any => :finalizing end + before_transition any => :finished do |migration| + migration.finished_at = Time.current if migration.respond_to?(:finished_at) + end + before_transition any => :active do |migration| migration.started_at = Time.current if migration.respond_to?(:started_at) end @@ -92,10 +96,6 @@ module Gitlab for_configuration(gitlab_schema, job_class_name, table_name, column_name, job_arguments).first end - def self.active_migration(connection:) - active_migrations_distinct_on_table(connection: connection, limit: 1).first - end - def self.find_executable(id, connection:) for_gitlab_schema(Gitlab::Database.gitlab_schemas_for_connection(connection)) .executable.find_by_id(id) @@ -220,7 +220,12 @@ module Gitlab end def health_context - HealthStatus::Context.new(connection, [table_name], gitlab_schema.to_sym) + @health_context ||= Gitlab::Database::HealthStatus::Context.new( + self, + connection, + [table_name], + gitlab_schema.to_sym + ) end def hold!(until_time: 10.minutes.from_now) diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 7224ff2b517..e3e8754c758 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -144,7 +144,7 @@ module Gitlab end def adjust_migration(active_migration) - signals = HealthStatus.evaluate(active_migration) + signals = Gitlab::Database::HealthStatus.evaluate(active_migration.health_context) if signals.any?(&:stop?) active_migration.hold! diff --git a/lib/gitlab/database/background_migration/health_status.rb b/lib/gitlab/database/background_migration/health_status.rb deleted file mode 100644 index 96905fd0bc5..00000000000 --- a/lib/gitlab/database/background_migration/health_status.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - module HealthStatus - DEFAULT_INIDICATORS = [ - Indicators::AutovacuumActiveOnTable, - Indicators::WriteAheadLog, - Indicators::PatroniApdex - ].freeze - - # Rather than passing along the migration, we use a more explicitly defined context - Context = Struct.new(:connection, :tables, :gitlab_schema) - - def self.evaluate(migration, indicators = DEFAULT_INIDICATORS) - indicators.map do |indicator| - signal = begin - indicator.new(migration.health_context).evaluate - rescue StandardError => e - Gitlab::ErrorTracking.track_exception(e, migration_id: migration.id, - job_class_name: migration.job_class_name) - - Signals::Unknown.new(indicator, reason: "unexpected error: #{e.message} (#{e.class})") - end - - log_signal(signal, migration) if signal.log_info? - - signal - end - end - - def self.log_signal(signal, migration) - Gitlab::BackgroundMigration::Logger.info( - migration_id: migration.id, - health_status_indicator: signal.indicator_class.to_s, - indicator_signal: signal.short_name, - signal_reason: signal.reason, - message: "#{migration} signaled: #{signal}" - ) - end - end - end - end -end diff --git a/lib/gitlab/database/background_migration/health_status/indicators.rb b/lib/gitlab/database/background_migration/health_status/indicators.rb deleted file mode 100644 index 69503e5b61f..00000000000 --- a/lib/gitlab/database/background_migration/health_status/indicators.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - module HealthStatus - module Indicators - end - end - end - end -end diff --git a/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb b/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb deleted file mode 100644 index 48e12609a13..00000000000 --- a/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - module HealthStatus - module Indicators - class AutovacuumActiveOnTable - def initialize(context) - @context = context - end - - def evaluate - return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? - - autovacuum_active_on = active_autovacuums_for(context.tables) - - if autovacuum_active_on.empty? - Signals::Normal.new(self.class, reason: 'no autovacuum running on any relevant tables') - else - Signals::Stop.new(self.class, reason: "autovacuum running on: #{autovacuum_active_on.join(', ')}") - end - end - - private - - attr_reader :context - - def enabled? - Feature.enabled?(:batched_migrations_health_status_autovacuum, type: :ops) - end - - def active_autovacuums_for(tables) - Gitlab::Database::PostgresAutovacuumActivity.for_tables(tables) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb b/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb deleted file mode 100644 index 0dd6dd5c2a4..00000000000 --- a/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb +++ /dev/null @@ -1,90 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - module HealthStatus - module Indicators - class PatroniApdex - include Gitlab::Utils::StrongMemoize - - def initialize(context) - @context = context - end - - def evaluate - return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? - - connection_error_message = fetch_connection_error_message - return unknown_signal(connection_error_message) if connection_error_message.present? - - apdex_sli = fetch_sli(apdex_sli_query) - return unknown_signal('Patroni service apdex can not be calculated') unless apdex_sli.present? - - if apdex_sli.to_f > apdex_slo.to_f - Signals::Normal.new(self.class, reason: 'Patroni service apdex is above SLO') - else - Signals::Stop.new(self.class, reason: 'Patroni service apdex is below SLO') - end - end - - private - - attr_reader :context - - def enabled? - Feature.enabled?(:batched_migrations_health_status_patroni_apdex, type: :ops) - end - - def unknown_signal(reason) - Signals::Unknown.new(self.class, reason: reason) - end - - def fetch_connection_error_message - return 'Patroni Apdex Settings not configured' unless database_apdex_settings.present? - return 'Prometheus client is not ready' unless client.ready? - return 'Apdex SLI query is not configured' unless apdex_sli_query - return 'Apdex SLO is not configured' unless apdex_slo - end - - def client - @client ||= Gitlab::PrometheusClient.new( - database_apdex_settings[:prometheus_api_url], - allow_local_requests: true, - verify: true - ) - end - - def database_apdex_settings - @database_apdex_settings ||= Gitlab::CurrentSettings.database_apdex_settings&.with_indifferent_access - end - - def apdex_sli_query - { - gitlab_main: database_apdex_settings[:apdex_sli_query][:main], - gitlab_ci: database_apdex_settings[:apdex_sli_query][:ci] - }.fetch(context.gitlab_schema.to_sym) - end - strong_memoize_attr :apdex_sli_query - - def apdex_slo - { - gitlab_main: database_apdex_settings[:apdex_slo][:main], - gitlab_ci: database_apdex_settings[:apdex_slo][:ci] - }.fetch(context.gitlab_schema.to_sym) - end - strong_memoize_attr :apdex_slo - - def fetch_sli(query) - response = client.query(query) - metric = response&.first || {} - value = metric.fetch('value', []) - - Array.wrap(value).second - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/background_migration/health_status/indicators/write_ahead_log.rb b/lib/gitlab/database/background_migration/health_status/indicators/write_ahead_log.rb deleted file mode 100644 index d2fb0a8b751..00000000000 --- a/lib/gitlab/database/background_migration/health_status/indicators/write_ahead_log.rb +++ /dev/null @@ -1,74 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - module HealthStatus - module Indicators - class WriteAheadLog - include Gitlab::Utils::StrongMemoize - - LIMIT = 42 - PENDING_WAL_COUNT_SQL = <<~SQL - WITH - current_wal_file AS ( - SELECT pg_walfile_name(pg_current_wal_insert_lsn()) AS pg_walfile_name - ), - current_wal AS ( - SELECT - ('x' || substring(pg_walfile_name, 9, 8))::bit(32)::int AS log, - ('x' || substring(pg_walfile_name, 17, 8))::bit(32)::int AS seg, - pg_walfile_name - FROM current_wal_file - ), - archive_wal AS ( - SELECT - ('x' || substring(last_archived_wal, 9, 8))::bit(32)::int AS log, - ('x' || substring(last_archived_wal, 17, 8))::bit(32)::int AS seg, - last_archived_wal - FROM pg_stat_archiver - ) - SELECT ((current_wal.log - archive_wal.log) * 256) + (current_wal.seg - archive_wal.seg) AS pending_wal_count - FROM current_wal, archive_wal - SQL - - def initialize(context) - @connection = context.connection - end - - def evaluate - return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? - - unless pending_wal_count - return Signals::NotAvailable.new(self.class, reason: 'WAL archive queue can not be calculated') - end - - if pending_wal_count > LIMIT - Signals::Stop.new(self.class, reason: "WAL archive queue is too big") - else - Signals::Normal.new(self.class, reason: 'WAL archive queue is within limit') - end - end - - private - - attr_reader :connection - - def enabled? - Feature.enabled?(:batched_migrations_health_status_wal, type: :ops) - end - - # Returns number of WAL segments pending archival - def pending_wal_count - strong_memoize(:pending_wal_count) do - Gitlab::Database::LoadBalancing::Session.current.use_primary do - connection.execute(PENDING_WAL_COUNT_SQL).to_a.first&.fetch('pending_wal_count') - end - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/background_migration/health_status/signals.rb b/lib/gitlab/database/background_migration/health_status/signals.rb deleted file mode 100644 index 534c4330cf2..00000000000 --- a/lib/gitlab/database/background_migration/health_status/signals.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module BackgroundMigration - module HealthStatus - module Signals - # Base class for a signal - class Base - attr_reader :indicator_class, :reason - - def initialize(indicator_class, reason:) - @indicator_class = indicator_class - @reason = reason - end - - def to_s - "#{short_name} (indicator: #{indicator_class}; reason: #{reason})" - end - - # :nocov: - def log_info? - false - end - - def stop? - false - end - # :nocov: - - def short_name - self.class.name.demodulize - end - end - - # A Signals::Stop is an indication to put a migration on hold or stop it entirely: - # In general, we want to slow down or pause the migration. - class Stop < Base - # :nocov: - def log_info? - true - end - - def stop? - true - end - # :nocov: - end - - # A Signals::Normal indicates normal system state: We carry on with the migration - # and may even attempt to optimize its throughput etc. - class Normal < Base; end - - # When given an Signals::Unknown, something unexpected happened while - # we evaluated system indicators. - class Unknown < Base - # :nocov: - def log_info? - true - end - # :nocov: - end - - # No signal could be determined, e.g. because the indicator - # was disabled. - class NotAvailable < Base; end - end - end - end - end -end diff --git a/lib/gitlab/database/convert_feature_category_to_group_label.rb b/lib/gitlab/database/convert_feature_category_to_group_label.rb new file mode 100644 index 00000000000..5a4599312ba --- /dev/null +++ b/lib/gitlab/database/convert_feature_category_to_group_label.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class ConvertFeatureCategoryToGroupLabel + STAGES_URL = 'https://gitlab.com/gitlab-com/www-gitlab-com/-/raw/master/data/stages.yml' + + def initialize(feature_category) + @feature_category = feature_category + end + + def execute + feature_categories_map[feature_category] + end + + private + + attr_reader :feature_category + + def stages + response = Gitlab::HTTP.get(STAGES_URL) + + YAML.safe_load(response) if response.success? + end + + def feature_categories_map + stages['stages'].each_with_object({}) do |(_, stage), result| + stage['groups'].each do |group_name, group| + group['categories'].each do |category| + result[category] = "group::#{group_name.sub('_', ' ')}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/database_connection_info.rb b/lib/gitlab/database/database_connection_info.rb new file mode 100644 index 00000000000..57ecbcd64ae --- /dev/null +++ b/lib/gitlab/database/database_connection_info.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Database + DatabaseConnectionInfo = Struct.new( + :name, + :description, + :gitlab_schemas, + :klass, + :fallback_database, + :db_dir, + :uses_load_balancing, + :file_path, + keyword_init: true + ) do + include Gitlab::Utils::StrongMemoize + + def initialize(*) + super + self.name = name.to_sym + self.gitlab_schemas = gitlab_schemas.map(&:to_sym) + self.klass = klass.constantize + self.fallback_database = fallback_database&.to_sym + self.db_dir = Rails.root.join(db_dir || 'db') + end + + def self.load_file(yaml_file) + content = YAML.load_file(yaml_file) + new(**content.deep_symbolize_keys.merge(file_path: yaml_file)) + end + + def active_record_base? + klass == ActiveRecord::Base + end + private :active_record_base? + + strong_memoize_attr def connection_class + klass.connection_class || active_record_base? ? klass : nil + end + + strong_memoize_attr def order + # Retain order of configurations as they are defined in `config/database.yml` + ActiveRecord::Base.configurations + .configs_for(env_name: Rails.env) + .map(&:name) + .index(name.to_s) || 1_000 + end + + def connection_class_or_fallback(all_databases) + if connection_class + connection_class + elsif fallback_database + all_databases.fetch(fallback_database) + .connection_class_or_fallback(all_databases) + end + end + + def has_gitlab_shared? + gitlab_schemas.include?(:gitlab_shared) + end + + def uses_load_balancing? + !!uses_load_balancing + end + + def db_docs_dir + db_dir.join('docs') + end + end + end +end diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index 02f008abf85..b1af62e4875 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -18,6 +18,7 @@ module Gitlab end end end + alias_method :each_db_connection, :each_database_connection def each_model_connection(models, only_on: nil, &blk) selected_databases = Array.wrap(only_on).map(&:to_sym) diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 4394c089b22..9b58284b389 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -3,13 +3,15 @@ # This module gathers information about table to schema mapping # to understand table affinity # -# Each table / view needs to have assigned gitlab_schema. Names supported today: +# Each table / view needs to have assigned gitlab_schema. For example: # # - gitlab_shared - defines a set of tables that are found on all databases (data accessed is dependent on connection) # - gitlab_main / gitlab_ci - defines a set of tables that can only exist on a given application database # - gitlab_geo - defines a set of tables that can only exist on the geo database # - gitlab_internal - defines all internal tables of Rails and PostgreSQL # +# All supported GitLab schemas can be viewed in `db/gitlab_schemas/` and `ee/db/gitlab_schemas/` +# # Tables for the purpose of tests should be prefixed with `_test_my_table_name` module Gitlab @@ -17,8 +19,6 @@ module Gitlab module GitlabSchema UnknownSchemaError = Class.new(StandardError) - DICTIONARY_PATH = 'db/docs/' - def self.table_schemas!(tables) tables.map { |table| table_schema!(table) }.to_set end @@ -67,44 +67,50 @@ module Gitlab # All `pg_` tables are marked as `internal` return :gitlab_internal if table_name.start_with?('pg_') - - # Sometimes the name of an index can be interpreted as a table's name. - # For eg, if we execute "ALTER INDEX my_index..", my_index is interpreted as a table name. - # In such cases, we should return the schema of the database table actually - # holding that index. - index_name = table_name - derive_schema_from_index(index_name) end # rubocop:enable Metrics/CyclomaticComplexity - def self.dictionary_path_globs - [Rails.root.join(DICTIONARY_PATH, '*.yml')] + def self.table_schema!(name) + # rubocop:disable Gitlab/DocUrl + self.table_schema(name) || raise( + UnknownSchemaError, + "Could not find gitlab schema for table #{name}: Any new or deleted tables must be added to the database dictionary " \ + "See https://docs.gitlab.com/ee/development/database/database_dictionary.html" + ) + # rubocop:enable Gitlab/DocUrl end - def self.view_path_globs - [Rails.root.join(DICTIONARY_PATH, 'views', '*.yml')] + private_class_method def self.cross_access_allowed?(type, table_schemas) + table_schemas.any? do |schema| + extra_schemas = table_schemas - [schema] + extra_schemas -= Gitlab::Database.all_gitlab_schemas[schema]&.public_send(type) || [] # rubocop:disable GitlabSecurity/PublicSend + extra_schemas.empty? + end end - def self.deleted_views_path_globs - [Rails.root.join(DICTIONARY_PATH, 'deleted_views', '*.yml')] + def self.cross_joins_allowed?(table_schemas) + table_schemas.empty? || self.cross_access_allowed?(:allow_cross_joins, table_schemas) end - def self.deleted_tables_path_globs - [Rails.root.join(DICTIONARY_PATH, 'deleted_tables', '*.yml')] + def self.cross_transactions_allowed?(table_schemas) + table_schemas.empty? || self.cross_access_allowed?(:allow_cross_transactions, table_schemas) end - def self.views_and_tables_to_schema - @views_and_tables_to_schema ||= self.tables_to_schema.merge(self.views_to_schema) + def self.cross_foreign_key_allowed?(table_schemas) + self.cross_access_allowed?(:allow_cross_foreign_keys, table_schemas) end - def self.table_schema!(name) - # rubocop:disable Gitlab/DocUrl - self.table_schema(name) || raise( - UnknownSchemaError, - "Could not find gitlab schema for table #{name}: Any new or deleted tables must be added to the database dictionary " \ - "See https://docs.gitlab.com/ee/development/database/database_dictionary.html" - ) - # rubocop:enable Gitlab/DocUrl + def self.dictionary_paths + Gitlab::Database.all_database_connections + .values.map(&:db_docs_dir).uniq + end + + def self.dictionary_path_globs(scope) + self.dictionary_paths.map { |path| Rails.root.join(path, scope, '*.yml') } + end + + def self.views_and_tables_to_schema + @views_and_tables_to_schema ||= self.tables_to_schema.merge(self.views_to_schema) end def self.deleted_views_and_tables_to_schema @@ -112,36 +118,27 @@ module Gitlab end def self.deleted_tables_to_schema - @deleted_tables_to_schema ||= self.build_dictionary(self.deleted_tables_path_globs) + @deleted_tables_to_schema ||= self.build_dictionary('deleted_tables').to_h end def self.deleted_views_to_schema - @deleted_views_to_schema ||= self.build_dictionary(self.deleted_views_path_globs) + @deleted_views_to_schema ||= self.build_dictionary('deleted_views').to_h end def self.tables_to_schema - @tables_to_schema ||= self.build_dictionary(self.dictionary_path_globs) + @tables_to_schema ||= self.build_dictionary('').to_h end def self.views_to_schema - @views_to_schema ||= self.build_dictionary(self.view_path_globs) + @views_to_schema ||= self.build_dictionary('views').to_h end def self.schema_names @schema_names ||= self.views_and_tables_to_schema.values.to_set end - private_class_method def self.derive_schema_from_index(index_name) - index = Gitlab::Database::PostgresIndex.find_by(name: index_name, - schema: ApplicationRecord.connection.current_schema) - - return unless index - - table_schema(index.tablename) - end - - private_class_method def self.build_dictionary(path_globs) - Dir.glob(path_globs).each_with_object({}) do |file_path, dic| + def self.build_dictionary(scope) + Dir.glob(dictionary_path_globs(scope)).map do |file_path| data = YAML.load_file(file_path) key_name = data['table_name'] || data['view_name'] @@ -156,7 +153,7 @@ module Gitlab end # rubocop:enable Gitlab/DocUrl - dic[key_name] = data['gitlab_schema'].to_sym + [key_name, data['gitlab_schema'].to_sym] end end end diff --git a/lib/gitlab/database/gitlab_schema_info.rb b/lib/gitlab/database/gitlab_schema_info.rb new file mode 100644 index 00000000000..34b89cb9006 --- /dev/null +++ b/lib/gitlab/database/gitlab_schema_info.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Database + GitlabSchemaInfo = Struct.new( + :name, + :description, + :allow_cross_joins, + :allow_cross_transactions, + :allow_cross_foreign_keys, + :file_path, + keyword_init: true + ) do + def initialize(*) + super + self.name = name.to_sym + self.allow_cross_joins = allow_cross_joins&.map(&:to_sym)&.freeze + self.allow_cross_transactions = allow_cross_transactions&.map(&:to_sym)&.freeze + self.allow_cross_foreign_keys = allow_cross_foreign_keys&.map(&:to_sym)&.freeze + end + + def self.load_file(yaml_file) + content = YAML.load_file(yaml_file) + new(**content.deep_symbolize_keys.merge(file_path: yaml_file)) + end + end + end +end diff --git a/lib/gitlab/database/health_status.rb b/lib/gitlab/database/health_status.rb new file mode 100644 index 00000000000..69bb8a70afd --- /dev/null +++ b/lib/gitlab/database/health_status.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + DEFAULT_INIDICATORS = [ + Indicators::AutovacuumActiveOnTable, + Indicators::WriteAheadLog, + Indicators::PatroniApdex + ].freeze + + class << self + def evaluate(context, indicators = DEFAULT_INIDICATORS) + indicators.map do |indicator| + signal = begin + indicator.new(context).evaluate + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e, **context.status_checker_info) + + Signals::Unknown.new(indicator, reason: "unexpected error: #{e.message} (#{e.class})") + end + + log_signal(signal, context) if signal.log_info? + + signal + end + end + + def log_signal(signal, context) + Gitlab::Database::HealthStatus::Logger.info(**context.status_checker_info.merge( + health_status_indicator: signal.indicator_class.to_s, + indicator_signal: signal.short_name, + signal_reason: signal.reason, + message: "#{context.status_checker} signaled: #{signal}" + )) + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/context.rb b/lib/gitlab/database/health_status/context.rb new file mode 100644 index 00000000000..717257a84ad --- /dev/null +++ b/lib/gitlab/database/health_status/context.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + class Context + attr_reader :status_checker, :connection, :tables, :gitlab_schema + + # status_checker: the caller object which checks for database health status + # eg: BackgroundMigration::BatchedMigration or DeferJobs::DatabaseHealthStatusChecker + def initialize(status_checker, connection, tables, gitlab_schema) + @status_checker = status_checker + @connection = connection + @tables = tables + @gitlab_schema = gitlab_schema + end + + def status_checker_info + { + status_checker_id: status_checker.id, + status_checker_type: status_checker.class.name, + job_class_name: status_checker.job_class_name + } + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/indicators.rb b/lib/gitlab/database/health_status/indicators.rb new file mode 100644 index 00000000000..a149c36aae4 --- /dev/null +++ b/lib/gitlab/database/health_status/indicators.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Indicators + end + end + end +end diff --git a/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb b/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb new file mode 100644 index 00000000000..6bf2bbf0c70 --- /dev/null +++ b/lib/gitlab/database/health_status/indicators/autovacuum_active_on_table.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Indicators + class AutovacuumActiveOnTable + def initialize(context) + @tables = context.tables + end + + def evaluate + return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? + + autovacuum_active_on = active_autovacuums_for(tables) + + if autovacuum_active_on.empty? + Signals::Normal.new(self.class, reason: 'no autovacuum running on any relevant tables') + else + Signals::Stop.new(self.class, reason: "autovacuum running on: #{autovacuum_active_on.join(', ')}") + end + end + + private + + attr_reader :tables + + def enabled? + Feature.enabled?(:batched_migrations_health_status_autovacuum, type: :ops) + end + + def active_autovacuums_for(tables) + Gitlab::Database::PostgresAutovacuumActivity.for_tables(tables) + end + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/indicators/patroni_apdex.rb b/lib/gitlab/database/health_status/indicators/patroni_apdex.rb new file mode 100644 index 00000000000..680c86cf7b2 --- /dev/null +++ b/lib/gitlab/database/health_status/indicators/patroni_apdex.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Indicators + class PatroniApdex + include Gitlab::Utils::StrongMemoize + + def initialize(context) + @gitlab_schema = context.gitlab_schema.to_sym + end + + def evaluate + return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? + + connection_error_message = fetch_connection_error_message + return unknown_signal(connection_error_message) if connection_error_message.present? + + apdex_sli = fetch_sli(apdex_sli_query) + return unknown_signal('Patroni service apdex can not be calculated') unless apdex_sli.present? + + if apdex_sli.to_f > apdex_slo.to_f + Signals::Normal.new(self.class, reason: 'Patroni service apdex is above SLO') + else + Signals::Stop.new(self.class, reason: 'Patroni service apdex is below SLO') + end + end + + private + + attr_reader :gitlab_schema + + def enabled? + Feature.enabled?(:batched_migrations_health_status_patroni_apdex, type: :ops) + end + + def unknown_signal(reason) + Signals::Unknown.new(self.class, reason: reason) + end + + def fetch_connection_error_message + return 'Patroni Apdex Settings not configured' unless database_apdex_settings.present? + return 'Prometheus client is not ready' unless client.ready? + return 'Apdex SLI query is not configured' unless apdex_sli_query + return 'Apdex SLO is not configured' unless apdex_slo + end + + def client + @client ||= Gitlab::PrometheusClient.new( + database_apdex_settings[:prometheus_api_url], + allow_local_requests: true, + verify: true + ) + end + + def database_apdex_settings + @database_apdex_settings ||= Gitlab::CurrentSettings.database_apdex_settings&.with_indifferent_access + end + + def apdex_sli_query + { + gitlab_main: database_apdex_settings[:apdex_sli_query][:main], + gitlab_ci: database_apdex_settings[:apdex_sli_query][:ci] + }.fetch(gitlab_schema) + end + strong_memoize_attr :apdex_sli_query + + def apdex_slo + { + gitlab_main: database_apdex_settings[:apdex_slo][:main], + gitlab_ci: database_apdex_settings[:apdex_slo][:ci] + }.fetch(gitlab_schema) + end + strong_memoize_attr :apdex_slo + + def fetch_sli(query) + response = client.query(query) + metric = response&.first || {} + value = metric.fetch('value', []) + + Array.wrap(value).second + end + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/indicators/write_ahead_log.rb b/lib/gitlab/database/health_status/indicators/write_ahead_log.rb new file mode 100644 index 00000000000..1614b17df48 --- /dev/null +++ b/lib/gitlab/database/health_status/indicators/write_ahead_log.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Indicators + class WriteAheadLog + include Gitlab::Utils::StrongMemoize + + LIMIT = 42 + PENDING_WAL_COUNT_SQL = <<~SQL + WITH + current_wal_file AS ( + SELECT pg_walfile_name(pg_current_wal_insert_lsn()) AS pg_walfile_name + ), + current_wal AS ( + SELECT + ('x' || substring(pg_walfile_name, 9, 8))::bit(32)::int AS log, + ('x' || substring(pg_walfile_name, 17, 8))::bit(32)::int AS seg, + pg_walfile_name + FROM current_wal_file + ), + archive_wal AS ( + SELECT + ('x' || substring(last_archived_wal, 9, 8))::bit(32)::int AS log, + ('x' || substring(last_archived_wal, 17, 8))::bit(32)::int AS seg, + last_archived_wal + FROM pg_stat_archiver + ) + SELECT ((current_wal.log - archive_wal.log) * 256) + (current_wal.seg - archive_wal.seg) AS pending_wal_count + FROM current_wal, archive_wal + SQL + + def initialize(context) + @connection = context.connection + end + + def evaluate + return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? + + unless pending_wal_count + return Signals::NotAvailable.new(self.class, reason: 'WAL archive queue can not be calculated') + end + + if pending_wal_count > LIMIT + Signals::Stop.new(self.class, reason: "WAL archive queue is too big") + else + Signals::Normal.new(self.class, reason: 'WAL archive queue is within limit') + end + end + + private + + attr_reader :connection + + def enabled? + Feature.enabled?(:batched_migrations_health_status_wal, type: :ops) + end + + # Returns number of WAL segments pending archival + def pending_wal_count + Gitlab::Database::LoadBalancing::Session.current.use_primary do + connection.execute(PENDING_WAL_COUNT_SQL).to_a.first&.fetch('pending_wal_count') + end + end + strong_memoize_attr :pending_wal_count + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/logger.rb b/lib/gitlab/database/health_status/logger.rb new file mode 100644 index 00000000000..820c1aeb695 --- /dev/null +++ b/lib/gitlab/database/health_status/logger.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + class Logger < ::Gitlab::JsonLogger + exclude_context! + + def self.file_name_noext + 'database_health_status' + end + end + end + end +end diff --git a/lib/gitlab/database/health_status/signals.rb b/lib/gitlab/database/health_status/signals.rb new file mode 100644 index 00000000000..e1bcdcae9c7 --- /dev/null +++ b/lib/gitlab/database/health_status/signals.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module HealthStatus + module Signals + # Base class for a signal + class Base + attr_reader :indicator_class, :reason + + def initialize(indicator_class, reason:) + @indicator_class = indicator_class + @reason = reason + end + + def to_s + "#{short_name} (indicator: #{indicator_class}; reason: #{reason})" + end + + def log_info? + false + end + + def stop? + false + end + + def short_name + self.class.name.demodulize + end + end + + # A Signals::Stop is an indication to put a migration on hold or stop it entirely: + # In general, we want to slow down or pause the migration. + class Stop < Base + def log_info? + true + end + + def stop? + true + end + end + + # A Signals::Normal indicates normal system state: We carry on with the migration + # and may even attempt to optimize its throughput etc. + class Normal < Base; end + + # When given an Signals::Unknown, something unexpected happened while + # we evaluated system indicators. + class Unknown < Base + def log_info? + true + end + end + + # No signal could be determined, e.g. because the indicator + # was disabled. + class NotAvailable < Base; end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/host.rb b/lib/gitlab/database/load_balancing/host.rb index bdbb80d6f31..f8ed5fcd4cc 100644 --- a/lib/gitlab/database/load_balancing/host.rb +++ b/lib/gitlab/database/load_balancing/host.rb @@ -16,6 +16,43 @@ module Gitlab PG::Error ].freeze + # This query checks that the current user has permissions before we try and query logical replication status. We + # also only allow >= PG14 because these views are only accessible to superuser before PG14 even if the + # has_table_privilege says otherwise. + CAN_TRACK_LOGICAL_LSN_QUERY = <<~SQL.squish.freeze + SELECT + has_table_privilege('pg_replication_origin_status', 'select') + AND + has_function_privilege('pg_show_replication_origin_status()', 'execute') + AND current_setting('server_version_num', true)::int >= 140000 + AS allowed + SQL + + # The following is necessary to handle a mix of logical and physical replicas. We assume that if they have + # pg_replication_origin_status then they are a logical replica. In a logical replica we need to use + # `remote_lsn` rather than `pg_last_wal_replay_lsn` in order for our LSN to be comparable to the source + # cluster. This logic would be broken if we have 2 logical subscriptions or if we have a logical subscription + # in the source primary cluster. Read more at https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121621 + LATEST_LSN_WITH_LOGICAL_QUERY = <<~SQL.squish.freeze + CASE + WHEN (SELECT TRUE FROM pg_replication_origin_status) THEN + (SELECT remote_lsn FROM pg_replication_origin_status) + WHEN pg_is_in_recovery() THEN + pg_last_wal_replay_lsn() + ELSE + pg_current_wal_insert_lsn() + END + SQL + + LATEST_LSN_WITHOUT_LOGICAL_QUERY = <<~SQL.squish.freeze + CASE + WHEN pg_is_in_recovery() THEN + pg_last_wal_replay_lsn() + ELSE + pg_current_wal_insert_lsn() + END + SQL + # host - The address of the database. # load_balancer - The LoadBalancer that manages this Host. def initialize(host, load_balancer, port: nil) @@ -30,6 +67,7 @@ module Gitlab @online = true @last_checked_at = Time.zone.now + # Randomly somewhere in between interval and 2*interval we'll refresh the status of the host interval = load_balancer.configuration.replica_check_interval @intervals = (interval..(interval * 2)).step(0.5).to_a end @@ -91,6 +129,7 @@ module Gitlab end def refresh_status + @latest_lsn_query = nil # Periodically clear the cached @latest_lsn_query value in case permissions change @online = replica_is_up_to_date? @last_checked_at = Time.zone.now end @@ -142,11 +181,11 @@ module Gitlab # primary. # # This method will return nil if no lag size could be calculated. - def replication_lag_size - location = connection.quote(primary_write_location) + def replication_lag_size(location = primary_write_location) + location = connection.quote(location) + row = query_and_release(<<-SQL.squish) - SELECT pg_wal_lsn_diff(#{location}, pg_last_wal_replay_lsn())::float - AS diff + SELECT pg_wal_lsn_diff(#{location}, (#{latest_lsn_query}))::float AS diff SQL row['diff'].to_i if row.any? @@ -173,22 +212,8 @@ module Gitlab # # location - The transaction write location as reported by a primary. def caught_up?(location) - string = connection.quote(location) - - # In case the host is a primary pg_last_wal_replay_lsn/pg_last_xlog_replay_location() returns - # NULL. The recovery check ensures we treat the host as up-to-date in - # such a case. - query = <<-SQL.squish - SELECT NOT pg_is_in_recovery() - OR pg_wal_lsn_diff(pg_last_wal_replay_lsn(), #{string}) >= 0 - AS result - SQL - - row = query_and_release(query) - - ::Gitlab::Utils.to_boolean(row['result']) - rescue *CONNECTION_ERRORS - false + lag = replication_lag_size(location) + lag.present? && lag.to_i <= 0 end def query_and_release(sql) @@ -198,6 +223,22 @@ module Gitlab ensure release_connection end + + private + + def can_track_logical_lsn? + row = query_and_release(CAN_TRACK_LOGICAL_LSN_QUERY) + + ::Gitlab::Utils.to_boolean(row['allowed']) + rescue *CONNECTION_ERRORS + false + end + + # The LATEST_LSN_WITH_LOGICAL query requires permissions that may not be present in self-managed configurations. + # We fallback gracefully to the query that does not correctly handle logical replicas for such configurations. + def latest_lsn_query + @latest_lsn_query ||= can_track_logical_lsn? ? LATEST_LSN_WITH_LOGICAL_QUERY : LATEST_LSN_WITHOUT_LOGICAL_QUERY + end end end end diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index 43e71e6bda2..8ddd871f93c 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -51,10 +51,15 @@ module Gitlab execute_sql_statement(sql_statement) - result_hash(action: 'locked') + result_hash(action: dry_run ? 'needs_lock' : 'locked') end def unlock_writes + unless table_locked_for_writes? + logger&.info "Skipping unlock_writes, because #{table_name} is already unlocked for writes" + return result_hash(action: 'skipped') + end + logger&.info "Database: '#{database_name}', Table: '#{table_name}': Allow Writes".color(:green) sql_statement = <<~SQL DROP TRIGGER IF EXISTS #{write_trigger_name} ON #{table_name}; @@ -62,7 +67,7 @@ module Gitlab execute_sql_statement(sql_statement) - result_hash(action: 'unlocked') + result_hash(action: dry_run ? 'needs_unlock' : 'unlocked') end private diff --git a/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb b/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb new file mode 100644 index 00000000000..555efb58606 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module WraparoundAutovacuum + # This is used for partitioning CI tables because the autovacuum for wraparound + # prevention can take many hours to complete on some of the tables and this in + # turn blocks the post deployment migrations pipeline. + # Intended workflow for this helper: + # 1. Introduce a migration that is guarded with this helper + # 2. Check that the migration was successfully executed on .com + # 3. Introduce the migration again for self-managed. + # + def can_execute_on?(*tables) + return false unless Gitlab.com? || Gitlab.dev_or_test_env? + + if wraparound_prevention_on_tables?(tables) + Gitlab::AppLogger.info(message: "Wraparound prevention vacuum detected", class: self.class) + say "Wraparound prevention vacuum detected, skipping migration" + return false + end + + true + end + + def wraparound_prevention_on_tables?(tables) + Gitlab::Database::PostgresAutovacuumActivity.reset_column_information + + Gitlab::Database::PostgresAutovacuumActivity + .wraparound_prevention + .for_tables(tables) + .any? + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/constraints_helpers.rb b/lib/gitlab/database/migrations/constraints_helpers.rb index 5aafc9f1444..1771f1f0fa7 100644 --- a/lib/gitlab/database/migrations/constraints_helpers.rb +++ b/lib/gitlab/database/migrations/constraints_helpers.rb @@ -262,6 +262,16 @@ module Gitlab SQL end + def switch_constraint_names(table_name, constraint_a, constraint_b) + validate_not_in_transaction!(:switch_constraint_names) + + with_lock_retries do + rename_constraint(table_name, constraint_a, :temp_name_for_renaming) + rename_constraint(table_name, constraint_b, constraint_a) + rename_constraint(table_name, :temp_name_for_renaming, constraint_b) + end + end + def validate_check_constraint_name!(constraint_name) return unless constraint_name.to_s.length > MAX_IDENTIFIER_NAME_LENGTH diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index 7222f148b3f..9895a68ec8d 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -40,7 +40,7 @@ module Gitlab next if model < ::Gitlab::Database::SharedModel && !(model < TableWithoutModel) model_connection_name = model.connection_db_config.name - Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| + Gitlab::Database::EachDatabase.each_db_connection(include_shared: false) do |connection, connection_name| if connection_name != model_connection_name PartitionManager.new(model, connection: connection).sync_partitions end @@ -60,6 +60,8 @@ module Gitlab end def drop_detached_partitions + return unless Feature.enabled?(:partition_manager_sync_partitions, type: :ops) + Gitlab::AppLogger.info(message: 'Dropping detached postgres partitions') Gitlab::Database::EachDatabase.each_database_connection do diff --git a/lib/gitlab/database/partitioning/list/convert_table.rb b/lib/gitlab/database/partitioning/list/convert_table.rb index d4fb150d956..9889d01be76 100644 --- a/lib/gitlab/database/partitioning/list/convert_table.rb +++ b/lib/gitlab/database/partitioning/list/convert_table.rb @@ -11,12 +11,11 @@ module Gitlab PARTITIONING_CONSTRAINT_NAME = 'partitioning_constraint' - attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value, - :locking_configuration + attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value def initialize( migration_context:, table_name:, parent_table_name:, partitioning_column:, - zero_partition_value:, lock_tables: []) + zero_partition_value:) @migration_context = migration_context @connection = migration_context.connection @@ -24,7 +23,6 @@ module Gitlab @parent_table_name = parent_table_name @partitioning_column = partitioning_column @zero_partition_value = zero_partition_value - @locking_configuration = LockingConfiguration.new(migration_context, table_locking_order: lock_tables) end def prepare_for_partitioning(async: false) @@ -38,22 +36,24 @@ module Gitlab end def partition - assert_existing_constraints_partitionable - assert_partitioning_constraint_present - - create_parent_table - attach_foreign_keys_to_parent - locking_sql = locking_configuration.locking_statement_for(tables_that_will_lock_during_partitioning) - - locking_configuration.with_lock_retries do - # Loose FKs trigger will exclusively lock the table and it might - # not follow the locking order needed to attach the partition. - migration_context.execute(locking_sql) if locking_sql.present? - - redefine_loose_foreign_key_triggers do - migration_context.execute(sql_to_convert_table) + # If already partitioned, the table is no longer partitionable. Thus we skip checks leading up + # to partitioning if the partitioning transaction has already succeeded. + unless already_partitioned? + assert_existing_constraints_partitionable + assert_partitioning_constraint_present + + create_parent_table + + migration_context.with_lock_retries do + redefine_loose_foreign_key_triggers do + migration_context.execute(sql_to_convert_table) + end end end + + # Attaching foreign keys handles cases where one or more foreign keys already exists, so it doesn't + # need a check similar to the rest of this method + attach_foreign_keys_to_parent end def revert_partitioning @@ -194,6 +194,8 @@ module Gitlab # At this point no other connection knows about the parent table. # Thus the only contended lock in the following transaction is on fk.to_table. # So a deadlock is impossible. + # (We also take a share update exclusive lock against the recently attached child table, + # but that only blocks vacuum and other schema modifications, not reads or writes) # If we're rerunning this migration after a failure to acquire a lock, the foreign key might already exist # Don't try to recreate it in that case @@ -299,16 +301,8 @@ module Gitlab end end - def tables_that_will_lock_during_partitioning - # Locks are taken against the table + all tables that reference it by foreign key - # postgres_foreign_keys.referenced_table_name gives the table name that we need here directly, but that - # column did not exist yet during the migration 20221021145820_create_routing_table_for_builds_metadata_v2 - # To ensure compatibility with that migration if it is run with this code, use referenced_table_identifier - # here. - referenced_tables = Gitlab::Database::PostgresForeignKey - .by_constrained_table_identifier(table_identifier) - .map { |fk| table_name_for_identifier(fk.referenced_table_identifier) } - referenced_tables + [table_name] + def already_partitioned? + Gitlab::Database::PostgresPartition.for_identifier(table_identifier).exists? end end end diff --git a/lib/gitlab/database/partitioning/sliding_list_strategy.rb b/lib/gitlab/database/partitioning/sliding_list_strategy.rb index 5bb34a86d43..8f8afdfc551 100644 --- a/lib/gitlab/database/partitioning/sliding_list_strategy.rb +++ b/lib/gitlab/database/partitioning/sliding_list_strategy.rb @@ -77,6 +77,19 @@ module Gitlab end def validate_and_fix + unless model.connection_db_config.name == + Gitlab::Database.db_config_name(Gitlab::Database::SharedModel.connection) + + Gitlab::AppLogger.warn( + message: 'Skipping fixing column default because connections mismatch', + event: :partition_manager_validate_and_fix_connection_mismatch, + model_connection_name: Gitlab::Database.db_config_name(model.connection), + shared_connection_name: Gitlab::Database.db_config_name(Gitlab::Database::SharedModel.connection) + ) + + return + end + return if no_partitions_exist? old_default_value = current_default_value diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 61e95dbe1a4..1ce0a44e37f 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -287,8 +287,7 @@ module Gitlab table_name: table_name, parent_table_name: parent_table_name, partitioning_column: partitioning_column, - zero_partition_value: initial_partitioning_value, - lock_tables: lock_tables + zero_partition_value: initial_partitioning_value ).partition end diff --git a/lib/gitlab/database/postgres_autovacuum_activity.rb b/lib/gitlab/database/postgres_autovacuum_activity.rb index a4dc199c259..6a80c631abb 100644 --- a/lib/gitlab/database/postgres_autovacuum_activity.rb +++ b/lib/gitlab/database/postgres_autovacuum_activity.rb @@ -6,9 +6,14 @@ module Gitlab self.table_name = 'postgres_autovacuum_activity' self.primary_key = 'table_identifier' + scope :wraparound_prevention, -> { where(wraparound_prevention: true) } + def self.for_tables(tables) Gitlab::Database::LoadBalancing::Session.current.use_primary do - where('schema = current_schema()').where(table: tables) + # calling `.to_a` here to execute the query in the primary's scope + # and to avoid having the scope chained and re-executed + # + where('schema = current_schema()').where(table: tables).to_a end end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index d15a0eaa44c..b5a45eed3a4 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -5,6 +5,7 @@ module Gitlab module QueryAnalyzers class PreventCrossDatabaseModification < Database::QueryAnalyzers::Base CrossDatabaseModificationAcrossUnsupportedTablesError = Class.new(QueryAnalyzerError) + QUERY_LIMIT = 10 # This method will allow cross database modifications within the block # Example: @@ -42,7 +43,8 @@ module Gitlab context.merge!({ transaction_depth_by_db: Hash.new { |h, k| h[k] = 0 }, modified_tables_by_db: Hash.new { |h, k| h[k] = Set.new }, - ignored_tables: [] + ignored_tables: [], + queries: [] }) end @@ -71,6 +73,7 @@ module Gitlab context[:transaction_depth_by_db][database] -= 1 if context[:transaction_depth_by_db][database] == 0 context[:modified_tables_by_db][database].clear + clear_queries # Attempt to troubleshoot https://gitlab.com/gitlab-org/gitlab/-/issues/351531 ::CrossDatabaseModification::TransactionStackTrackRecord.log_gitlab_transactions_stack(action: :end_of_transaction) @@ -104,22 +107,25 @@ module Gitlab # databases return if tables == ['schema_migrations'] + add_to_queries(sql) context[:modified_tables_by_db][database].merge(tables) all_tables = context[:modified_tables_by_db].values.flat_map(&:to_a) schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(all_tables) - schemas += ApplicationRecord.gitlab_transactions_stack - if schemas.many? - message = "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ - "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " \ - "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception." + unless ::Gitlab::Database::GitlabSchema.cross_transactions_allowed?(schemas) + messages = [] + messages << "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ + "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " + messages << "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions " \ + "for details on how to resolve this exception." + messages += cleaned_queries - raise CrossDatabaseModificationAcrossUnsupportedTablesError, message + raise CrossDatabaseModificationAcrossUnsupportedTablesError, messages.join("\n\n") end rescue CrossDatabaseModificationAcrossUnsupportedTablesError => e ::Gitlab::ErrorTracking.track_exception(e, { gitlab_schemas: schemas, tables: all_tables, query: parsed.sql }) - raise if raise_exception? + raise if dev_or_test_env? end # rubocop:enable Metrics/AbcSize @@ -159,12 +165,28 @@ module Gitlab end end - # We only raise in tests for now otherwise some features will be broken - # in development. For now we've mostly only added allowlist based on - # spec names. Until we have allowed all the violations inline we don't - # want to raise in development. - def self.raise_exception? - Rails.env.test? + def self.dev_or_test_env? + Gitlab.dev_or_test_env? + end + + def self.clear_queries + return unless dev_or_test_env? + + context[:queries].clear + end + + def self.add_to_queries(sql) + return unless dev_or_test_env? + + context[:queries].push(sql) + end + + def self.cleaned_queries + return [] unless dev_or_test_env? + + context[:queries].last(QUERY_LIMIT).each_with_index.map do |sql, i| + "#{i}: #{sql}" + end end def self.in_transaction? diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb index f3e0fc26946..4e1ab700542 100644 --- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -19,7 +19,8 @@ module Gitlab gitlab_internal: nil, # Pods specific changes - gitlab_main_clusterwide: :gitlab_main + gitlab_main_clusterwide: :gitlab_main, + gitlab_main_cell: :gitlab_main }.freeze class << self @@ -61,6 +62,7 @@ module Gitlab def restrict_to_ddl_only(parsed) tables = self.dml_tables(parsed) schemas = self.dml_schemas(tables) + schemas = self.map_schemas(schemas) if schemas.any? self.raise_dml_not_allowed_error("Modifying of '#{tables}' (#{schemas.to_a}) with '#{parsed.sql}'") @@ -78,8 +80,10 @@ module Gitlab tables = self.dml_tables(parsed) schemas = self.dml_schemas(tables) + schemas = self.map_schemas(schemas) + allowed_schemas = self.map_schemas(self.allowed_gitlab_schemas) - if (schemas - self.allowed_gitlab_schemas).any? + if (schemas - allowed_schemas).any? raise DMLAccessDeniedError, \ "Select/DML queries (SELECT/UPDATE/DELETE) do access '#{tables}' (#{schemas.to_a}) " \ "which is outside of list of allowed schemas: '#{self.allowed_gitlab_schemas}'. " \ @@ -100,15 +104,19 @@ module Gitlab end def dml_schemas(tables) - extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(tables) + ::Gitlab::Database::GitlabSchema.table_schemas!(tables) + end + + def map_schemas(schemas) + schemas = schemas.to_set - SCHEMA_MAPPING.each do |schema, mapped_schema| - next unless extra_schemas.delete?(schema) + SCHEMA_MAPPING.each do |in_schema, mapped_schema| + next unless schemas.delete?(in_schema) - extra_schemas.add(mapped_schema) if mapped_schema + schemas.add(mapped_schema) if mapped_schema end - extra_schemas + schemas end def raise_dml_not_allowed_error(message) diff --git a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb index 420195d89dd..20814b098c1 100644 --- a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb +++ b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb @@ -104,7 +104,7 @@ module Gitlab when :func_call "#{parse_node(node.func_call.funcname.first)}()" when :a_const - parse_node(node.a_const.val) + parse_a_const(node.a_const) when :type_cast value = parse_node(node.type_cast.arg) type = type(node.type_cast.type_name) @@ -112,10 +112,21 @@ module Gitlab [MAPPINGS.fetch(value, "'#{value}'"), separator].compact.join('') else - node.to_h[node.node].values.last + get_value_from_key(node, key: node.node) end end + def parse_a_const(a_const) + return unless a_const + + type = a_const.val + get_value_from_key(a_const, key: type) + end + + def get_value_from_key(node, key:) + node.to_h[key].values.last + end + def partition_keys return [] unless partitioning_stmt diff --git a/lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb b/lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb new file mode 100644 index 00000000000..3b45f5c77ca --- /dev/null +++ b/lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Adapters + class ForeignKeyDatabaseAdapter + def initialize(query_result) + @query_result = query_result + end + + def name + "#{query_result['schema']}.#{query_result['foreign_key_name']}" + end + + def table_name + query_result['table_name'] + end + + def statement + query_result['foreign_key_definition'] + end + + private + + attr_reader :query_result + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb b/lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb new file mode 100644 index 00000000000..e4c1e1adab3 --- /dev/null +++ b/lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Adapters + class ForeignKeyStructureSqlAdapter + STATEMENT_REGEX = /\bREFERENCES\s\K\S+\K\s\(/ + EXTRACT_REGEX = /\bFOREIGN KEY.*/ + + def initialize(parsed_stmt) + @parsed_stmt = parsed_stmt + end + + def name + "#{schema_name}.#{foreign_key_name}" + end + + def table_name + parsed_stmt.relation.relname + end + + # PgQuery parses FK statements with an extra space in the referenced table column. + # This extra space needs to be removed. + # + # @example REFERENCES ci_pipelines (id) => REFERENCES ci_pipelines(id) + def statement + deparse_stmt[EXTRACT_REGEX].gsub(STATEMENT_REGEX, '(') + end + + private + + attr_reader :parsed_stmt + + def schema_name + parsed_stmt.relation.schemaname + end + + def foreign_key_name + parsed_stmt.cmds.first.alter_table_cmd.def.constraint.conname + end + + def deparse_stmt + PgQuery.deparse_stmt(parsed_stmt) + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/database.rb b/lib/gitlab/database/schema_validation/database.rb index 7a0e348a27b..858bf618f44 100644 --- a/lib/gitlab/database/schema_validation/database.rb +++ b/lib/gitlab/database/schema_validation/database.rb @@ -18,6 +18,10 @@ module Gitlab trigger_map[trigger_name] end + def fetch_foreign_key_by_name(foreign_key_name) + foreign_key_map[foreign_key_name] + end + def fetch_table_by_name(table_name) table_map[table_name] end @@ -30,6 +34,10 @@ module Gitlab trigger_map[trigger_name].present? end + def foreign_key_exists?(foreign_key_name) + fetch_foreign_key_by_name(foreign_key_name).present? + end + def table_exists?(table_name) fetch_table_by_name(table_name).present? end @@ -42,6 +50,10 @@ module Gitlab trigger_map.values end + def foreign_keys + foreign_key_map.values + end + def tables table_map.values end @@ -68,6 +80,14 @@ module Gitlab end end + def foreign_key_map + @foreign_key_map ||= fetch_fks.each_with_object({}) do |stmt, result| + adapter = Adapters::ForeignKeyDatabaseAdapter.new(stmt) + + result[adapter.name] = SchemaObjects::ForeignKey.new(adapter) + end + end + def table_map @table_map ||= fetch_tables.transform_values! do |stmt| columns = stmt.map { |column| SchemaObjects::Column.new(Adapters::ColumnDatabaseAdapter.new(column)) } @@ -122,6 +142,24 @@ module Gitlab connection.exec_query(sql, nil, schemas).group_by { |row| row['table_name'] } end + + def fetch_fks + sql = <<~SQL + SELECT + pg_namespace.nspname::text AS schema, + pg_class.relname::text AS table_name, + pg_constraint.conname AS foreign_key_name, + pg_get_constraintdef(pg_constraint.oid) AS foreign_key_definition + FROM pg_constraint + INNER JOIN pg_class ON pg_constraint.conrelid = pg_class.oid + INNER JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid + WHERE contype = 'f' + AND pg_namespace.nspname = $1 + AND pg_constraint.conparentid = 0 + SQL + + connection.exec_query(sql, nil, [connection.current_schema]) + end end end end diff --git a/lib/gitlab/database/schema_validation/schema_inconsistency.rb b/lib/gitlab/database/schema_validation/schema_inconsistency.rb index 6f50603e784..9f39db5b4c0 100644 --- a/lib/gitlab/database/schema_validation/schema_inconsistency.rb +++ b/lib/gitlab/database/schema_validation/schema_inconsistency.rb @@ -8,7 +8,9 @@ module Gitlab belongs_to :issue - validates :object_name, :valitador_name, :table_name, presence: true + validates :object_name, :valitador_name, :table_name, :diff, presence: true + + scope :with_open_issues, -> { joins(:issue).where('issue.state_id': Issue.available_states[:opened]) } end end end diff --git a/lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb b/lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb new file mode 100644 index 00000000000..b616b1a72b7 --- /dev/null +++ b/lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module SchemaObjects + class ForeignKey + def initialize(adapter) + @adapter = adapter + end + + # Foreign key name should include the schema, as the same name could be used across different schemas + # + # @example public.foreign_key_name + def name + @name ||= adapter.name + end + + def table_name + @table_name ||= adapter.table_name + end + + def statement + @statement ||= adapter.statement + end + + private + + attr_reader :adapter + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/structure_sql.rb b/lib/gitlab/database/schema_validation/structure_sql.rb index 299db1c2a7e..4d6fa17f0fc 100644 --- a/lib/gitlab/database/schema_validation/structure_sql.rb +++ b/lib/gitlab/database/schema_validation/structure_sql.rb @@ -19,6 +19,10 @@ module Gitlab triggers.find { |trigger| trigger.name == trigger_name }.present? end + def foreign_key_exists?(foreign_key_name) + foreign_keys.find { |fk| fk.name == foreign_key_name }.present? + end + def fetch_table_by_name(table_name) tables.find { |table| table.name == table_name } end @@ -35,6 +39,14 @@ module Gitlab @triggers ||= map_with_default_schema(trigger_statements, SchemaObjects::Trigger) end + def foreign_keys + @foreign_keys ||= foreign_key_statements.map do |stmt| + stmt.relation.schemaname = schema_name if stmt.relation.schemaname == '' + + SchemaObjects::ForeignKey.new(Adapters::ForeignKeyStructureSqlAdapter.new(stmt)) + end + end + def tables @tables ||= table_statements.map do |stmt| table_name = stmt.relation.relname @@ -65,6 +77,33 @@ module Gitlab statements.filter_map { |s| s.stmt.create_stmt } end + def foreign_key_statements + constraint_statements(:CONSTR_FOREIGN) + end + + # Filter constraint statement nodes + # + # @param constraint_type [Symbol] node type. One of CONSTR_PRIMARY, CONSTR_CHECK, CONSTR_EXCLUSION, + # CONSTR_UNIQUE or CONSTR_FOREIGN. + def constraint_statements(constraint_type) + alter_table_statements(:AT_AddConstraint).filter do |stmt| + stmt.cmds.first.alter_table_cmd.def.constraint.contype == constraint_type + end + end + + # Filter alter table statement nodes + # + # @param subtype [Symbol] node subtype +AT_AttachPartition+, +AT_ColumnDefault+ or +AT_AddConstraint+ + def alter_table_statements(subtype) + statements.filter_map do |statement| + node = statement.stmt.alter_table_stmt + + next unless node + + node if node.cmds.first.alter_table_cmd.subtype == subtype + end + end + def statements @statements ||= parsed_structure_file.tree.stmts end diff --git a/lib/gitlab/database/schema_validation/track_inconsistency.rb b/lib/gitlab/database/schema_validation/track_inconsistency.rb index 47c3492971c..6e167653d32 100644 --- a/lib/gitlab/database/schema_validation/track_inconsistency.rb +++ b/lib/gitlab/database/schema_validation/track_inconsistency.rb @@ -4,6 +4,8 @@ module Gitlab module Database module SchemaValidation class TrackInconsistency + COLUMN_TEXT_LIMIT = 6144 + def initialize(inconsistency, project, user) @inconsistency = inconsistency @project = project @@ -12,10 +14,13 @@ module Gitlab def execute return unless Gitlab.com? - return if inconsistency_record.present? + return refresh_issue if inconsistency_record.present? - result = ::Issues::CreateService.new(container: project, current_user: user, params: params, - spam_params: nil).execute + result = ::Issues::CreateService.new( + container: project, + current_user: user, + params: params, + perform_spam_check: false).execute track_inconsistency(result[:issue]) if result.success? end @@ -25,20 +30,21 @@ module Gitlab attr_reader :inconsistency, :project, :user def track_inconsistency(issue) - schema_inconsistency_model.create( + schema_inconsistency_model.create!( issue: issue, object_name: inconsistency.object_name, table_name: inconsistency.table_name, - valitador_name: inconsistency.type + valitador_name: inconsistency.type, + diff: inconsistency_diff ) end def params { title: issue_title, - description: issue_description, + description: description, issue_type: 'issue', - labels: %w[database database-inconsistency-report] + labels: default_labels + group_labels } end @@ -46,7 +52,7 @@ module Gitlab "New schema inconsistency: #{inconsistency.object_name}" end - def issue_description + def description <<~MSG We have detected a new schema inconsistency. @@ -81,12 +87,46 @@ module Gitlab MSG end + def group_labels + dictionary = YAML.safe_load(File.read(table_file_path)) + + dictionary['feature_categories'].to_a.filter_map do |feature_category| + Gitlab::Database::ConvertFeatureCategoryToGroupLabel.new(feature_category).execute + end + rescue Errno::ENOENT + [] + end + + def default_labels + %w[database database-inconsistency-report type::maintenance severity::4] + end + + def table_file_path + Rails.root.join(Gitlab::Database::GitlabSchema.dictionary_paths.first, "#{inconsistency.table_name}.yml") + end + def schema_inconsistency_model Gitlab::Database::SchemaValidation::SchemaInconsistency end + def refresh_issue + return if inconsistency_diff == inconsistency_record.diff # Nothing to refresh + + note = ::Notes::CreateService.new( + inconsistency_record.issue.project, + user, + { noteable_type: 'Issue', noteable: inconsistency_record.issue, note: description } + ).execute + + inconsistency_record.update!(diff: inconsistency_diff) if note.persisted? + end + + def inconsistency_diff + @inconsistency_diff ||= inconsistency.diff.to_s.first(COLUMN_TEXT_LIMIT) + end + def inconsistency_record - schema_inconsistency_model.find_by( + @inconsistency_record ||= schema_inconsistency_model.with_open_issues.find_by( object_name: inconsistency.object_name, table_name: inconsistency.table_name, valitador_name: inconsistency.type diff --git a/lib/gitlab/database/schema_validation/validators/base_validator.rb b/lib/gitlab/database/schema_validation/validators/base_validator.rb index 58e0bf5292b..ee322e50a2c 100644 --- a/lib/gitlab/database/schema_validation/validators/base_validator.rb +++ b/lib/gitlab/database/schema_validation/validators/base_validator.rb @@ -18,13 +18,16 @@ module Gitlab ExtraTableColumns, ExtraIndexes, ExtraTriggers, + ExtraForeignKeys, MissingTables, MissingTableColumns, MissingIndexes, MissingTriggers, + MissingForeignKeys, DifferentDefinitionTables, DifferentDefinitionIndexes, - DifferentDefinitionTriggers + DifferentDefinitionTriggers, + DifferentDefinitionForeignKeys ] end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb b/lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb new file mode 100644 index 00000000000..8969fa76cd8 --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class DifferentDefinitionForeignKeys < BaseValidator + ERROR_MESSAGE = "The %s foreign key has a different statement between structure.sql and database" + + def execute + structure_sql.foreign_keys.filter_map do |structure_sql_fk| + database_fk = database.fetch_foreign_key_by_name(structure_sql_fk.name) + + next if database_fk.nil? + next if database_fk.statement == structure_sql_fk.statement + + build_inconsistency(self.class, structure_sql_fk, database_fk) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb b/lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb new file mode 100644 index 00000000000..887e86c7bfd --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class ExtraForeignKeys < BaseValidator + ERROR_MESSAGE = "The foreign key %s is present in the database, but not in the structure.sql file" + + def execute + database.foreign_keys.filter_map do |database_fk| + next if structure_sql.foreign_key_exists?(database_fk.name) + + build_inconsistency(self.class, nil, database_fk) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb b/lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb new file mode 100644 index 00000000000..b20f8474426 --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class MissingForeignKeys < BaseValidator + ERROR_MESSAGE = "The foreign key %s is missing from the database" + + def execute + structure_sql.foreign_keys.filter_map do |structure_sql_fk| + next if database.foreign_key_exists?(structure_sql_fk.name) + + build_inconsistency(self.class, structure_sql_fk, nil) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb index 0b0d46f4b0e..02e0da022f9 100644 --- a/lib/gitlab/database/tables_locker.rb +++ b/lib/gitlab/database/tables_locker.rb @@ -5,10 +5,11 @@ module Gitlab class TablesLocker GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo].freeze - def initialize(logger: nil, dry_run: false) + def initialize(logger: nil, dry_run: false, include_partitions: true) @logger = logger @dry_run = dry_run @result = [] + @include_partitions = include_partitions end def unlock_writes @@ -50,6 +51,7 @@ module Gitlab # Unlocks the writes on the table and its partitions def unlock_writes_on_table(table_name, connection, database_name) @result << lock_writes_manager(table_name, connection, database_name).unlock_writes + return unless @include_partitions table_attached_partitions(table_name, connection) do |postgres_partition| @result << lock_writes_manager(postgres_partition.identifier, connection, database_name).unlock_writes @@ -59,6 +61,7 @@ module Gitlab # It locks the writes on the table and its partitions def lock_writes_on_table(table_name, connection, database_name) @result << lock_writes_manager(table_name, connection, database_name).lock_writes + return unless @include_partitions table_attached_partitions(table_name, connection) do |postgres_partition| @result << lock_writes_manager(postgres_partition.identifier, connection, database_name).lock_writes @@ -67,6 +70,7 @@ module Gitlab def tables_to_lock(connection, &block) Gitlab::Database::GitlabSchema.tables_to_schema.each(&block) + return unless @include_partitions Gitlab::Database::SharedModel.using_connection(connection) do Postgresql::DetachedPartition.find_each do |detached_partition| diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index a6430d1758b..7ae1981fa2b 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -3,7 +3,7 @@ module Gitlab module Database class TablesTruncate - GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo].freeze + GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo gitlab_embedding].freeze def initialize(database_name:, min_batch_size:, logger: nil, until_table: nil, dry_run: false) @database_name = database_name diff --git a/lib/gitlab/database_importers/common_metrics.rb b/lib/gitlab/database_importers/common_metrics.rb deleted file mode 100644 index f964ae8a275..00000000000 --- a/lib/gitlab/database_importers/common_metrics.rb +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module CommonMetrics - end - end -end diff --git a/lib/gitlab/database_importers/common_metrics/importer.rb b/lib/gitlab/database_importers/common_metrics/importer.rb deleted file mode 100644 index 6c61e05674e..00000000000 --- a/lib/gitlab/database_importers/common_metrics/importer.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module CommonMetrics - class Importer - MissingQueryId = Class.new(StandardError) - - attr_reader :content - - def initialize(filename = 'common_metrics.yml') - @content = YAML.load_file(Rails.root.join('config', 'prometheus', filename)) - end - - def execute - CommonMetrics::PrometheusMetric.reset_column_information - - process_content do |id, attributes| - find_or_build_metric!(id) - .update!(**attributes) - end - end - - private - - def process_content(&blk) - content['panel_groups'].map do |group| - process_group(group, &blk) - end - end - - def process_group(group, &blk) - attributes = { - group: find_group_title_key(group['group']) - } - - group['panels'].map do |panel| - process_panel(panel, attributes, &blk) - end - end - - def process_panel(panel, attributes, &blk) - attributes = attributes.merge( - title: panel['title'], - y_label: panel['y_label']) - - panel['metrics'].map do |metric_details| - process_metric_details(metric_details, attributes, &blk) - end - end - - def process_metric_details(metric_details, attributes, &blk) - attributes = attributes.merge( - legend: metric_details['label'], - query: metric_details['query_range'], - unit: metric_details['unit']) - - yield(metric_details['id'], attributes) - end - - def find_or_build_metric!(id) - raise MissingQueryId unless id - - CommonMetrics::PrometheusMetric.common.find_by(identifier: id) || - CommonMetrics::PrometheusMetric.new(common: true, identifier: id) - end - - def find_group_title_key(title) - CommonMetrics::PrometheusMetricEnums.groups[find_group_title(title)] - end - - def find_group_title(title) - CommonMetrics::PrometheusMetricEnums.group_titles.invert[title] - end - end - end - end -end diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb deleted file mode 100644 index b4a392cbea9..00000000000 --- a/lib/gitlab/database_importers/common_metrics/prometheus_metric.rb +++ /dev/null @@ -1,12 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module CommonMetrics - class PrometheusMetric < ApplicationRecord - enum group: PrometheusMetricEnums.groups - scope :common, -> { where(common: true) } - end - end - end -end diff --git a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb b/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb deleted file mode 100644 index 8a5f53be20f..00000000000 --- a/lib/gitlab/database_importers/common_metrics/prometheus_metric_enums.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module CommonMetrics - module PrometheusMetricEnums - def self.groups - { - # built-in groups - nginx_ingress_vts: -1, - ha_proxy: -2, - aws_elb: -3, - nginx: -4, - kubernetes: -5, - nginx_ingress: -6, - - # custom groups - business: 0, - response: 1, - system: 2, - custom: 3, - - cluster_health: -100 - } - end - - def self.group_titles - { - business: _('Business metrics (Custom)'), - response: _('Response metrics (Custom)'), - system: _('System metrics (Custom)'), - nginx_ingress_vts: _('Response metrics (NGINX Ingress VTS)'), - nginx_ingress: _('Response metrics (NGINX Ingress)'), - ha_proxy: _('Response metrics (HA Proxy)'), - aws_elb: _('Response metrics (AWS ELB)'), - nginx: _('Response metrics (NGINX)'), - kubernetes: _('System metrics (Kubernetes)'), - cluster_health: _('Cluster Health'), - custom: _('Custom metrics') - } - end - end - end - end -end diff --git a/lib/gitlab/database_importers/default_organization_importer.rb b/lib/gitlab/database_importers/default_organization_importer.rb new file mode 100644 index 00000000000..147c0d19b01 --- /dev/null +++ b/lib/gitlab/database_importers/default_organization_importer.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module DefaultOrganizationImporter + def self.create_default_organization + return if Organizations::Organization.default_organization + + # When adding or changing attributes, consider changing the factory for Organization model as well + # spec/factories/organizations/organizations.rb + Organizations::Organization.create!( + id: Organizations::Organization::DEFAULT_ORGANIZATION_ID, + name: 'Default', + path: 'default' + ) + end + end + end +end diff --git a/lib/gitlab/dependency_linker/requirements_txt_linker.rb b/lib/gitlab/dependency_linker/requirements_txt_linker.rb index f630c13b760..dc654964e0b 100644 --- a/lib/gitlab/dependency_linker/requirements_txt_linker.rb +++ b/lib/gitlab/dependency_linker/requirements_txt_linker.rb @@ -9,7 +9,7 @@ module Gitlab def link_dependencies link_regex(/^(?<name>(?![a-z+]+:)[^#.-][^ ><=~!;\[]+)/) do |name| - "https://pypi.python.org/pypi/#{name}" + "https://pypi.org/project/#{name}/" end link_regex(%r{^(?<name>https?://[^ ]+)}, &:itself) diff --git a/lib/gitlab/devise_failure.rb b/lib/gitlab/devise_failure.rb index ffd057e1d33..2c48ad3b46b 100644 --- a/lib/gitlab/devise_failure.rb +++ b/lib/gitlab/devise_failure.rb @@ -7,6 +7,14 @@ module Gitlab def http_auth? request_format && super end + + def respond + if warden_options[:reason] == :too_many_requests + self.status = 403 + else + super + end + end end end diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb index 19fc028594c..807ce3682ab 100644 --- a/lib/gitlab/diff/formatters/base_formatter.rb +++ b/lib/gitlab/diff/formatters/base_formatter.rb @@ -9,6 +9,7 @@ module Gitlab attr_reader :base_sha attr_reader :start_sha attr_reader :head_sha + attr_reader :ignore_whitespace_change def initialize(attrs) if diff_file = attrs[:diff_file] diff --git a/lib/gitlab/diff/formatters/file_formatter.rb b/lib/gitlab/diff/formatters/file_formatter.rb new file mode 100644 index 00000000000..37b9ad85ef8 --- /dev/null +++ b/lib/gitlab/diff/formatters/file_formatter.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + module Formatters + class FileFormatter < BaseFormatter + def initialize(attrs) + @ignore_whitespace_change = false + + super(attrs) + end + + def key + @key ||= super.push(new_path, old_path) + end + + def position_type + "file" + end + + def complete? + [new_path, old_path].all?(&:present?) + end + + def ==(other) + other.is_a?(self.class) && + old_path == other.old_path && + new_path == other.new_path + end + end + end + end +end diff --git a/lib/gitlab/diff/formatters/image_formatter.rb b/lib/gitlab/diff/formatters/image_formatter.rb index d0c13dee1aa..f0d25885387 100644 --- a/lib/gitlab/diff/formatters/image_formatter.rb +++ b/lib/gitlab/diff/formatters/image_formatter.rb @@ -14,6 +14,7 @@ module Gitlab @y = attrs[:y] @width = attrs[:width] @height = attrs[:height] + @ignore_whitespace_change = false super(attrs) end diff --git a/lib/gitlab/diff/formatters/text_formatter.rb b/lib/gitlab/diff/formatters/text_formatter.rb index 9ea9bdfdf15..bd083720165 100644 --- a/lib/gitlab/diff/formatters/text_formatter.rb +++ b/lib/gitlab/diff/formatters/text_formatter.rb @@ -12,6 +12,7 @@ module Gitlab @old_line = attrs[:old_line] @new_line = attrs[:new_line] @line_range = attrs[:line_range] + @ignore_whitespace_change = !!attrs[:ignore_whitespace_change] super(attrs) end @@ -25,7 +26,8 @@ module Gitlab end def to_h - super.merge(old_line: old_line, new_line: new_line, line_range: line_range) + super.merge(old_line: old_line, new_line: new_line, line_range: line_range, + ignore_whitespace_change: ignore_whitespace_change) end def line_age diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 40b6ae2f14e..feee4bcc7f9 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -19,7 +19,8 @@ module Gitlab :x, :y, :line_range, - :position_type, to: :formatter + :position_type, + :ignore_whitespace_change, to: :formatter # A position can belong to a text line or to an image coordinate # it depends of the position_type argument. @@ -69,11 +70,11 @@ module Gitlab end def to_json(opts = nil) - Gitlab::Json.generate(formatter.to_h, opts) + Gitlab::Json.generate(to_h.except(:ignore_whitespace_change), opts) end def as_json(opts = nil) - to_h.as_json(opts) + to_h.except(:ignore_whitespace_change).as_json(opts) end def type @@ -134,7 +135,7 @@ module Gitlab end def diff_options - { paths: paths, expanded: true, include_stats: false } + { paths: paths, expanded: true, include_stats: false, ignore_whitespace_change: ignore_whitespace_change } end def diff_line(repository) @@ -149,6 +150,10 @@ module Gitlab @file_hash ||= Digest::SHA1.hexdigest(file_path) end + def on_file? + position_type == 'file' + end + def on_image? position_type == 'image' end @@ -184,6 +189,8 @@ module Gitlab case type when 'image' Gitlab::Diff::Formatters::ImageFormatter + when 'file' + Gitlab::Diff::Formatters::FileFormatter else Gitlab::Diff::Formatters::TextFormatter end diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index 1c21c35fa60..a8c0108fa34 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -21,9 +21,9 @@ module Gitlab return unless old_diff_refs&.complete? && new_diff_refs&.complete? return unless old_position.diff_refs == old_diff_refs - strategy = old_position.on_text? ? LineStrategy : ImageStrategy + @ignore_whitespace_change = old_position.ignore_whitespace_change - strategy.new(self).trace(old_position) + strategy(old_position).new(self).trace(old_position) end def ac_diffs @@ -48,9 +48,19 @@ module Gitlab private + def strategy(old_position) + if old_position.on_text? + LineStrategy + elsif old_position.on_file? + FileStrategy + else + ImageStrategy + end + end + def compare(start_sha, head_sha, straight: false) compare = CompareService.new(project, head_sha).execute(project, start_sha, straight: straight) - compare.diffs(paths: paths, expanded: true) + compare.diffs(paths: paths, expanded: true, ignore_whitespace_change: @ignore_whitespace_change) end end end diff --git a/lib/gitlab/diff/position_tracer/file_strategy.rb b/lib/gitlab/diff/position_tracer/file_strategy.rb new file mode 100644 index 00000000000..171d78bf46f --- /dev/null +++ b/lib/gitlab/diff/position_tracer/file_strategy.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class PositionTracer + class FileStrategy < BaseStrategy + def trace(position) + a_path = position.old_path + b_path = position.new_path + + # If file exists in B->D (e.g. updated, renamed, removed), let the + # note become outdated. + bd_diff = bd_diffs.diff_file_with_old_path(b_path) + + return { position: new_position(position, bd_diff), outdated: true } if bd_diff + + # If file still exists in the new diff, update the position. + cd_diff = cd_diffs.diff_file_with_new_path(b_path) + + return { position: new_position(position, cd_diff), outdated: false } if cd_diff + + # If file exists in A->C (e.g. rebased and same changes were present + # in target branch), let the note become outdated. + ac_diff = ac_diffs.diff_file_with_old_path(a_path) + + return { position: new_position(position, ac_diff), outdated: true } if ac_diff + + # If ever there's a case that the file no longer exists in any diff, + # don't set a change position and let the note become outdated. + # + # This should never happen given the file should exist in one of the + # diffs above. + { outdated: true } + end + + private + + def new_position(position, diff_file) + Position.new( + diff_file: diff_file, + position_type: position.position_type + ) + end + end + end + end +end diff --git a/lib/gitlab/diff/position_tracer/image_strategy.rb b/lib/gitlab/diff/position_tracer/image_strategy.rb index aac52b536f7..172eba4acd3 100644 --- a/lib/gitlab/diff/position_tracer/image_strategy.rb +++ b/lib/gitlab/diff/position_tracer/image_strategy.rb @@ -3,36 +3,7 @@ module Gitlab module Diff class PositionTracer - class ImageStrategy < BaseStrategy - def trace(position) - a_path = position.old_path - b_path = position.new_path - - # If file exists in B->D (e.g. updated, renamed, removed), let the - # note become outdated. - bd_diff = bd_diffs.diff_file_with_old_path(b_path) - - return { position: new_position(position, bd_diff), outdated: true } if bd_diff - - # If file still exists in the new diff, update the position. - cd_diff = cd_diffs.diff_file_with_new_path(b_path) - - return { position: new_position(position, cd_diff), outdated: false } if cd_diff - - # If file exists in A->C (e.g. rebased and same changes were present - # in target branch), let the note become outdated. - ac_diff = ac_diffs.diff_file_with_old_path(a_path) - - return { position: new_position(position, ac_diff), outdated: true } if ac_diff - - # If ever there's a case that the file no longer exists in any diff, - # don't set a change position and let the note become outdated. - # - # This should never happen given the file should exist in one of the - # diffs above. - { outdated: true } - end - + class ImageStrategy < FileStrategy private def new_position(position, diff_file) diff --git a/lib/gitlab/diff/position_tracer/line_strategy.rb b/lib/gitlab/diff/position_tracer/line_strategy.rb index d7a7e3f5425..0de9aa22008 100644 --- a/lib/gitlab/diff/position_tracer/line_strategy.rb +++ b/lib/gitlab/diff/position_tracer/line_strategy.rb @@ -62,6 +62,8 @@ module Gitlab # The line number as of D can be found by using the LineMapper on diff C->D # and providing the line number as of C. + @ignore_whitespace_change = position.ignore_whitespace_change + if position.added? trace_added_line(position) elsif position.removed? @@ -189,7 +191,8 @@ module Gitlab diff_file: diff_file, old_line: old_line, new_line: new_line, - line_range: line_range + line_range: line_range, + ignore_whitespace_change: @ignore_whitespace_change }.compact Position.new(**params) diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index df93e6e91b4..18ff7c28e17 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -16,7 +16,7 @@ module Gitlab def write_multiple(mapping) with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |pipelined| + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipelined| mapping.each do |raw_key, value| key = cache_key_for(raw_key) @@ -41,8 +41,8 @@ module Gitlab content = with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - if ::Feature.enabled?(:use_pipeline_over_multikey) - redis.pipelined do |pipeline| + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| keys.each { |key| pipeline.get(key) } end else @@ -72,10 +72,8 @@ module Gitlab with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - if ::Feature.enabled?(:use_pipeline_over_multikey) - redis.pipelined do |pipeline| - keys.each { |key| pipeline.del(key) } - end.sum + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::ClusterUtil.batch_unlink(keys, redis) else redis.del(keys) end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index c325112b673..869bcc6e2be 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -68,7 +68,7 @@ module Gitlab title: mail.subject, description: message_including_reply_or_only_quotes }, - spam_params: nil + perform_spam_check: false ).execute end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 076ba42daac..215ba77db13 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -103,7 +103,7 @@ module Gitlab cc: mail.cc } }, - spam_params: nil + perform_spam_check: false ).execute raise InvalidIssueError if result.error? diff --git a/lib/gitlab/email/hook/silent_mode_interceptor.rb b/lib/gitlab/email/hook/silent_mode_interceptor.rb index 56f94119472..774d4ac1f45 100644 --- a/lib/gitlab/email/hook/silent_mode_interceptor.rb +++ b/lib/gitlab/email/hook/silent_mode_interceptor.rb @@ -5,19 +5,17 @@ module Gitlab module Hook class SilentModeInterceptor def self.delivering_email(message) - if Gitlab::CurrentSettings.silent_mode_enabled? + if ::Gitlab::SilentMode.enabled? message.perform_deliveries = false - Gitlab::AppJsonLogger.info( + ::Gitlab::SilentMode.log_info( message: "SilentModeInterceptor prevented sending mail", - mail_subject: message.subject, - silent_mode_enabled: true + mail_subject: message.subject ) else - Gitlab::AppJsonLogger.debug( + ::Gitlab::SilentMode.log_debug( message: "SilentModeInterceptor did nothing", - mail_subject: message.subject, - silent_mode_enabled: false + mail_subject: message.subject ) end end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb index c2d645138d7..e7462b711f1 100644 --- a/lib/gitlab/email/reply_parser.rb +++ b/lib/gitlab/email/reply_parser.rb @@ -80,9 +80,7 @@ module Gitlab # # Plain email. # ``` - # So, we had to force its part to corresponding encoding before able - # to convert it to UTF-8 - force_utf8(object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8"))) + object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s else object.body.to_s end diff --git a/lib/gitlab/error_tracking/error_repository.rb b/lib/gitlab/error_tracking/error_repository.rb index fd2467add20..3871305c9c5 100644 --- a/lib/gitlab/error_tracking/error_repository.rb +++ b/lib/gitlab/error_tracking/error_repository.rb @@ -16,7 +16,7 @@ module Gitlab # @return [self] def self.build(project) strategy = - if Feature.enabled?(:use_click_house_database_for_error_tracking, project) + if Feature.enabled?(:gitlab_error_tracking, project) OpenApiStrategy.new(project) else ActiveRecordStrategy.new(project) diff --git a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb index e168fa10630..398ddebd355 100644 --- a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb +++ b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb @@ -127,21 +127,22 @@ module Gitlab def to_sentry_error(error) Gitlab::ErrorTracking::Error.new( id: error.fingerprint.to_s, - title: error.name, + title: "#{error.name}: #{error.description}", message: error.description, culprit: error.actor, first_seen: error.first_seen_at, last_seen: error.last_seen_at, status: error.status, count: error.event_count, - user_count: error.approximated_user_count + user_count: error.approximated_user_count, + frequency: error.stats&.frequency&.dig(:'24h') || [] ) end def to_sentry_detailed_error(error) Gitlab::ErrorTracking::DetailedError.new( id: error.fingerprint.to_s, - title: error.name, + title: "#{error.name}: #{error.description}", message: error.description, culprit: error.actor, first_seen: error.first_seen_at.to_s, @@ -155,7 +156,8 @@ module Gitlab external_base_url: external_base_url, integrated: true, first_release_version: release_from(oldest_event_for(error.fingerprint)), - last_release_version: release_from(newest_event_for(error.fingerprint)) + last_release_version: release_from(newest_event_for(error.fingerprint)), + frequency: error.stats&.frequency&.dig(:'24h') || [] ) end diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index 7aabf699a59..786a68c86f2 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -57,7 +57,7 @@ module Gitlab end def weak_etag_format(value) - %Q{W/"#{value}"} + %{W/"#{value}"} end def handle_cache_hit(etag, route, request) diff --git a/lib/gitlab/etag_caching/router/rails.rb b/lib/gitlab/etag_caching/router/rails.rb index 2924370f494..5fd592c43e4 100644 --- a/lib/gitlab/etag_caching/router/rails.rb +++ b/lib/gitlab/etag_caching/router/rails.rb @@ -17,7 +17,7 @@ module Gitlab new environments].freeze RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape))) - RESERVED_WORDS_PREFIX = %Q(^(?!.*\/(#{RESERVED_WORDS_REGEX})\/).*) + RESERVED_WORDS_PREFIX = %(^(?!.*\/(#{RESERVED_WORDS_REGEX})\/).*) ROUTES = [ [ diff --git a/lib/gitlab/front_matter.rb b/lib/gitlab/front_matter.rb index 2a759434b81..d215a77f4d7 100644 --- a/lib/gitlab/front_matter.rb +++ b/lib/gitlab/front_matter.rb @@ -37,6 +37,6 @@ module Gitlab # rubocop:enable Style/StringConcatenation PATTERN_UNTRUSTED_REGEX = - Gitlab::UntrustedRegexp.new(PATTERN_UNTRUSTED, multiline: true) + Gitlab::UntrustedRegexp.new(PATTERN_UNTRUSTED, multiline: true).freeze end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 80d0fd17568..ed45d3eb030 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -91,12 +91,10 @@ module Gitlab end # Default branch in the repository - def root_ref - gitaly_ref_client.default_branch_name - rescue GRPC::NotFound => e - raise NoRepository, e.message - rescue GRPC::Unknown => e - raise Gitlab::Git::CommandError, e.message + def root_ref(head_only: false) + wrapped_gitaly_errors do + gitaly_ref_client.default_branch_name(head_only: head_only) + end end def exists? @@ -520,13 +518,15 @@ module Gitlab empty_diff_stats end - def find_changed_paths(commits) + def find_changed_paths(commits, merge_commit_diff_mode: nil) processed_commits = commits.reject { |ref| ref.blank? || Gitlab::Git.blank_ref?(ref) } return [] if processed_commits.empty? wrapped_gitaly_errors do - gitaly_commit_client.find_changed_paths(processed_commits) + gitaly_commit_client.find_changed_paths( + processed_commits, merge_commit_diff_mode: merge_commit_diff_mode + ) end rescue CommandError, TypeError, NoRepository [] diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 37977a1dfb6..5b54ba472d9 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -76,8 +76,16 @@ module Gitlab encode! @message end - def tagger - @raw_tag.tagger + def user_name + encode! tagger.name if tagger + end + + def user_email + encode! tagger.email if tagger + end + + def date + Time.at(tagger.date.seconds).utc if tagger end def has_signature? @@ -105,6 +113,10 @@ module Gitlab private + def tagger + @raw_tag.tagger + end + def message_from_gitaly_tag return @raw_tag.message.dup if full_message_fetched_from_gitaly? diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index e437f99dab3..df3d8165ef2 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -6,7 +6,7 @@ module Gitlab include Gitlab::EncodingHelper extend Gitlab::Git::WrapsGitalyErrors - attr_accessor :id, :type, :mode, :commit_id, :submodule_url + attr_accessor :id, :type, :mode, :commit_id, :submodule_url, :ref_type attr_writer :name, :path, :flat_path class << self diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 0c67b9fa078..aa25fd3589a 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -12,6 +12,11 @@ module Gitlab 'unspecified' => Gitaly::CommitDiffRequest::WhitespaceChanges::WHITESPACE_CHANGES_UNSPECIFIED }.freeze + MERGE_COMMIT_DIFF_MODES = { + all_parents: Gitaly::FindChangedPathsRequest::MergeCommitDiffMode::MERGE_COMMIT_DIFF_MODE_ALL_PARENTS, + include_merges: Gitaly::FindChangedPathsRequest::MergeCommitDiffMode::MERGE_COMMIT_DIFF_MODE_INCLUDE_MERGES + }.freeze + TREE_ENTRIES_DEFAULT_LIMIT = 100_000 def initialize(repository) @@ -123,8 +128,10 @@ module Gitlab end def tree_entries(repository, revision, path, recursive, skip_flat_paths, pagination_params) - pagination_params ||= {} - pagination_params[:limit] ||= TREE_ENTRIES_DEFAULT_LIMIT + unless pagination_params.nil? && recursive + pagination_params ||= {} + pagination_params[:limit] ||= TREE_ENTRIES_DEFAULT_LIMIT + end request = Gitaly::GetTreeEntriesRequest.new( repository: @gitaly_repo, @@ -157,6 +164,17 @@ module Gitlab end [entries, cursor] + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error.try(:error) + when :path + raise Gitlab::Git::Index::IndexError, path_error_message(detailed_error.path) + when :resolve_tree + raise Gitlab::Git::Index::IndexError, e.details + else + raise e + end end def commit_count(ref, options = {}) @@ -229,11 +247,35 @@ module Gitlab response.flat_map { |rsp| rsp.stats.to_a } end - def find_changed_paths(commits) - request = Gitaly::FindChangedPathsRequest.new( - repository: @gitaly_repo, - commits: commits - ) + # When finding changed paths and passing a sha for a merge commit we can + # specify how to diff the commit. + # + # When diffing a merge commit and merge_commit_diff_mode is :all_parents + # file paths are only returned if changed in both parents (or all parents + # if diffing an octopus merge) + # + # This means if we create a merge request that includes a merge commit + # of changes already existing in the target branch, we can omit those + # changes when looking up the changed paths. + # + # e.g. + # 1. User branches from master to new branch named feature/foo_bar + # 2. User changes ./foo_bar.rb and commits change to feature/foo_bar + # 3. Another user merges a change to ./bar_baz.rb to master + # 4. User merges master into feature/foo_bar + # 5. User pushes to GitLab + # 6. GitLab checks which files have changed + # + # case merge_commit_diff_mode + # when :all_parents + # ['foo_bar.rb'] + # when :include_merges + # ['foo_bar.rb', 'bar_baz.rb'], + # else # defaults to :include_merges behavior + # ['foo_bar.rb', 'bar_baz.rb'], + # + def find_changed_paths(commits, merge_commit_diff_mode: nil) + request = find_changed_paths_request(commits, merge_commit_diff_mode) response = gitaly_client_call(@repository.storage, :diff_service, :find_changed_paths, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| @@ -595,6 +637,37 @@ module Gitlab response.commit end + + def find_changed_paths_request(commits, merge_commit_diff_mode) + diff_mode = MERGE_COMMIT_DIFF_MODES[merge_commit_diff_mode] if Feature.enabled?(:merge_commit_diff_modes) + + if Feature.disabled?(:find_changed_paths_new_format) + return Gitaly::FindChangedPathsRequest.new(repository: @gitaly_repo, commits: commits, merge_commit_diff_mode: diff_mode) + end + + commit_requests = commits.map do |commit| + Gitaly::FindChangedPathsRequest::Request.new( + commit_request: Gitaly::FindChangedPathsRequest::Request::CommitRequest.new(commit_revision: commit) + ) + end + + Gitaly::FindChangedPathsRequest.new(repository: @gitaly_repo, requests: commit_requests, merge_commit_diff_mode: diff_mode) + end + + def path_error_message(path_error) + case path_error.error_type + when :ERROR_TYPE_EMPTY_PATH + "You must provide a file path" + when :ERROR_TYPE_RELATIVE_PATH_ESCAPES_REPOSITORY + "Path cannot include traversal syntax" + when :ERROR_TYPE_ABSOLUTE_PATH + "Only relative path is accepted" + when :ERROR_TYPE_LONG_PATH + "Path is too long" + else + "Unknown path error" + end + end end end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 1af06cc7490..bd6cc9105d9 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -8,6 +8,8 @@ module Gitlab MAX_MSG_SIZE = 128.kilobytes.freeze + CUSTOM_HOOK_FALLBACK_MESSAGE = 'Prevented by server hooks' + def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository @@ -52,7 +54,7 @@ module Gitlab raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) when :custom_hook raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), - fallback_message: e.details) + fallback_message: CUSTOM_HOOK_FALLBACK_MESSAGE) when :reference_exists raise Gitlab::Git::Repository::TagExistsError else @@ -85,7 +87,7 @@ module Gitlab case detailed_error.try(:error) when :custom_hook raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), - fallback_message: e.details) + fallback_message: CUSTOM_HOOK_FALLBACK_MESSAGE) else if e.code == GRPC::Core::StatusCodes::FAILED_PRECONDITION raise Gitlab::Git::Repository::InvalidRef, e @@ -127,7 +129,7 @@ module Gitlab case detailed_error.try(:error) when :custom_hook raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), - fallback_message: e.details) + fallback_message: CUSTOM_HOOK_FALLBACK_MESSAGE) else raise end @@ -195,7 +197,7 @@ module Gitlab raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) when :custom_hook raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), - fallback_message: e.details) + fallback_message: CUSTOM_HOOK_FALLBACK_MESSAGE) when :reference_update # We simply ignore any reference update errors which are typically an # indicator of multiple RPC calls trying to update the same reference @@ -465,7 +467,7 @@ module Gitlab raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) when :custom_hook raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), - fallback_message: e.details) + fallback_message: CUSTOM_HOOK_FALLBACK_MESSAGE) when :index_update raise Gitlab::Git::Index::IndexError, index_error_message(detailed_error.index_update) else @@ -583,8 +585,7 @@ module Gitlab def custom_hook_error_message(custom_hook_error) # Custom hooks may return messages via either stdout or stderr which have a specific prefix. If - # that prefix is present we'll want to print the hook's output, otherwise we'll want to print the - # Gitaly error as a fallback. + # that prefix is present we'll want to print the hook's output. custom_hook_output = custom_hook_error.stderr.presence || custom_hook_error.stdout EncodingHelper.encode!(custom_hook_output) end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 88c79eb8954..45edfd4cbbf 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -50,8 +50,8 @@ module Gitlab consume_find_all_branches_response(response) end - def default_branch_name - request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) + def default_branch_name(head_only: false) + request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo, head_only: head_only) response = gitaly_client_call(@storage, :ref_service, :find_default_branch_name, request, timeout: GitalyClient.fast_timeout) Gitlab::Git.branch_name(response.name) end diff --git a/lib/gitlab/github_gists_import/importer/gist_importer.rb b/lib/gitlab/github_gists_import/importer/gist_importer.rb index 4018f425e7c..71dfe5e2aa5 100644 --- a/lib/gitlab/github_gists_import/importer/gist_importer.rb +++ b/lib/gitlab/github_gists_import/importer/gist_importer.rb @@ -4,10 +4,13 @@ module Gitlab module GithubGistsImport module Importer class GistImporter - attr_reader :gist, :user + attr_reader :gist, :user, :snippet FileCountLimitError = Class.new(StandardError) + RepoSizeLimitError = Class.new(StandardError) + SnippetRepositoryError = Class.new(StandardError) FILE_COUNT_LIMIT_MESSAGE = 'Snippet maximum file count exceeded' + REPO_SIZE_LIMIT_MESSAGE = 'Snippet repository size exceeded' # gist - An instance of `Gitlab::GithubGistsImport::Representation::Gist`. def initialize(gist, user_id) @@ -16,12 +19,15 @@ module Gitlab end def execute - snippet = build_snippet - import_repository(snippet) if snippet.save! + validate_gist! - return ServiceResponse.success unless max_snippet_files_count_exceeded?(snippet) + @snippet = build_snippet + import_repository if snippet.save! + validate_repository! - fail_and_track(snippet) + ServiceResponse.success + rescue FileCountLimitError, RepoSizeLimitError, SnippetRepositoryError => exception + fail_and_track(snippet, exception) end private @@ -40,13 +46,13 @@ module Gitlab PersonalSnippet.new(attrs) end - def import_repository(snippet) + def import_repository resolved_address = get_resolved_address snippet.create_repository snippet.repository.fetch_as_mirror(gist.git_pull_url, forced: true, resolved_address: resolved_address) rescue StandardError - remove_snippet_and_repository(snippet) + remove_snippet_and_repository raise end @@ -61,11 +67,19 @@ module Gitlab host.present? ? validated_pull_url.host.to_s : '' end - def max_snippet_files_count_exceeded?(snippet) - snippet.all_files.size > Snippet.max_file_limit + def check_gist_files_count! + return if gist.files.count <= Snippet.max_file_limit + + raise FileCountLimitError, FILE_COUNT_LIMIT_MESSAGE end - def remove_snippet_and_repository(snippet) + def check_gist_repo_size! + return if gist.total_files_size <= Gitlab::CurrentSettings.snippet_size_limit + + raise RepoSizeLimitError, REPO_SIZE_LIMIT_MESSAGE + end + + def remove_snippet_and_repository snippet.repository.remove if snippet.repository_exists? snippet.destroy end @@ -74,10 +88,21 @@ module Gitlab Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end - def fail_and_track(snippet) - remove_snippet_and_repository(snippet) + def fail_and_track(snippet, exception) + remove_snippet_and_repository if snippet + + ServiceResponse.error(message: exception.message).track_exception(as: exception.class) + end + + def validate_gist! + check_gist_files_count! + check_gist_repo_size! + end + + def validate_repository! + result = Snippets::RepositoryValidationService.new(user, snippet).execute - ServiceResponse.error(message: FILE_COUNT_LIMIT_MESSAGE).track_exception(as: FileCountLimitError) + raise SnippetRepositoryError, result.message if result.error? end end end diff --git a/lib/gitlab/github_gists_import/representation/gist.rb b/lib/gitlab/github_gists_import/representation/gist.rb index 0d309a98f38..674da4f3400 100644 --- a/lib/gitlab/github_gists_import/representation/gist.rb +++ b/lib/gitlab/github_gists_import/representation/gist.rb @@ -65,6 +65,10 @@ module Gitlab def github_identifiers { id: id } end + + def total_files_size + files.values.sum { |f| f[:size].to_i } + end end end end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 2654812b64a..d37942aa8a3 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -54,6 +54,8 @@ module Gitlab project.change_head(default_branch) if default_branch + validate_repository_size! + # The initial fetch can bring in lots of loose refs and objects. # Running a `git gc` will make importing pull requests faster. Repositories::HousekeepingService.new(project, :gc).execute @@ -89,7 +91,13 @@ module Gitlab strong_memoize_attr def client_repository client.repository(project.import_source) end + + def validate_repository_size! + # Defined in EE + end end end end end + +Gitlab::GithubImport::Importer::RepositoryImporter.prepend_mod diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index 0408b34bb02..191e15962a6 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -12,7 +12,7 @@ module Gitlab expose_attribute :noteable_id, :commit_id, :file_path, :diff_hunk, :author, :created_at, :updated_at, :original_commit_id, :note_id, :end_line, :start_line, - :side, :in_reply_to_id, :discussion_id + :side, :in_reply_to_id, :discussion_id, :subject_type # Builds a diff note from a GitHub API response. # @@ -43,7 +43,8 @@ module Gitlab start_line: note[:start_line], side: note[:side], in_reply_to_id: note[:in_reply_to_id], - discussion_id: DiffNotes::DiscussionId.new(note).find_or_generate + discussion_id: DiffNotes::DiscussionId.new(note).find_or_generate, + subject_type: note[:subject_type] } new(hash) @@ -84,8 +85,14 @@ module Gitlab end def line_code - diff_line = Gitlab::Diff::Parser.new.parse(diff_hunk.lines).to_a.last + # on the GitHub side it is possible to leave a comment on a file + # or on a line. When the comment is left on a file there is no + # diff hunk, but LegacyDiffNote requires line_code to be always present + # and DiffFile requires it for text files + # so it is set as the first line for any type of file (image, binary, text) + return Gitlab::Git.diff_line_code(file_path, 1, 1) if on_file? + diff_line = Gitlab::Diff::Parser.new.parse(diff_hunk.lines).to_a.last Gitlab::Git.diff_line_code(file_path, diff_line.new_pos, diff_line.old_pos) end @@ -141,6 +148,10 @@ module Gitlab def addition? side == 'RIGHT' end + + def on_file? + subject_type == 'file' + end end end end diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index 268d5d3e564..9161a1a138f 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -34,7 +34,7 @@ module Gitlab DESIGN = ::Gitlab::GlRepository::RepoType.new( name: :design, access_checker_class: ::Gitlab::GitAccessDesign, - repository_resolver: -> (project) { project.design_management_repository.repository }, + repository_resolver: -> (project) { project.find_or_create_design_management_repository.repository }, suffix: :design, container_class: DesignManagement::Repository ).freeze diff --git a/lib/gitlab/gl_repository/identifier.rb b/lib/gitlab/gl_repository/identifier.rb index f521a14ea19..787e80fb763 100644 --- a/lib/gitlab/gl_repository/identifier.rb +++ b/lib/gitlab/gl_repository/identifier.rb @@ -23,7 +23,7 @@ module Gitlab return identifier if identifier&.valid? - raise InvalidIdentifier, %Q(Invalid GL Repository "#{gl_repository}") + raise InvalidIdentifier, %(Invalid GL Repository "#{gl_repository}") end # The older 2-segment format, where the container is implied. diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 904a2ccc79b..9eeea7336b5 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -50,8 +50,12 @@ module Gitlab gon.jh = Gitlab.jh? gon.dot_com = Gitlab.com? gon.uf_error_prefix = ::Gitlab::Utils::ErrorMessage::UF_ERROR_PREFIX + gon.pat_prefix = Gitlab::CurrentSettings.current_application_settings.personal_access_token_prefix + + gon.diagramsnet_url = Gitlab::CurrentSettings.diagramsnet_url if Gitlab::CurrentSettings.diagramsnet_enabled if current_user + gon.version = Gitlab::VERSION # publish version only for logged in users gon.current_user_id = current_user.id gon.current_username = current_user.username gon.current_user_fullname = current_user.name @@ -66,10 +70,11 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) - push_frontend_feature_flag(:super_sidebar_peek, current_user) push_frontend_feature_flag(:unbatch_graphql_queries, current_user) + push_frontend_feature_flag(:command_palette, current_user) # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248 push_frontend_feature_flag(:remove_monitor_metrics) + push_frontend_feature_flag(:gitlab_duo, current_user) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/generic_tracing.rb b/lib/gitlab/graphql/generic_tracing.rb index d3de9c714f4..dc3f6574631 100644 --- a/lib/gitlab/graphql/generic_tracing.rb +++ b/lib/gitlab/graphql/generic_tracing.rb @@ -39,11 +39,15 @@ module Gitlab ensure duration = Gitlab::Metrics::System.monotonic_time - start - graphql_duration_seconds.observe(tags, duration) + graphql_duration_seconds.observe(tags, duration) unless deactivated? end private + def deactivated? + Feature.enabled?(:graphql_generic_tracing_metrics_deactivate) + end + def with_labkit_tracing(tags, &block) return yield unless Labkit::Tracing.enabled? diff --git a/lib/gitlab/hotlinking_detector.rb b/lib/gitlab/hotlinking_detector.rb index b5000777010..e81983cd014 100644 --- a/lib/gitlab/hotlinking_detector.rb +++ b/lib/gitlab/hotlinking_detector.rb @@ -25,6 +25,11 @@ module Gitlab return true if INVALID_FORMATS.include?(request_accepts.first) false + + rescue ActionDispatch::Http::MimeNegotiation::InvalidType, Mime::Type::InvalidMimeType + # Malformed requests with invalid MIME types prevent the checks from + # being executed correctly, so we should intercept those requests. + true end private diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index c6cd5fbfced..8b19611e5c0 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -10,6 +10,7 @@ module Gitlab RedirectionTooDeep = Class.new(StandardError) ReadTotalTimeout = Class.new(Net::ReadTimeout) HeaderReadTimeout = Class.new(Net::ReadTimeout) + SilentModeBlockedError = Class.new(StandardError) HTTP_TIMEOUT_ERRORS = [ Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Gitlab::HTTP::ReadTotalTimeout @@ -28,6 +29,13 @@ module Gitlab }.freeze DEFAULT_READ_TOTAL_TIMEOUT = 30.seconds + SILENT_MODE_ALLOWED_METHODS = [ + Net::HTTP::Get, + Net::HTTP::Head, + Net::HTTP::Options, + Net::HTTP::Trace + ].freeze + include HTTParty # rubocop:disable Gitlab/HTTParty class << self @@ -37,6 +45,8 @@ module Gitlab connection_adapter HTTPConnectionAdapter def self.perform_request(http_method, path, options, &block) + raise_if_blocked_by_silent_mode(http_method) + log_info = options.delete(:extra_log_info) options_with_timeouts = if !options.has_key?(:timeout) @@ -76,5 +86,20 @@ module Gitlab rescue *HTTP_ERRORS nil end + + def self.raise_if_blocked_by_silent_mode(http_method) + return unless blocked_by_silent_mode?(http_method) + + ::Gitlab::SilentMode.log_info( + message: 'Outbound HTTP request blocked', + outbound_http_request_method: http_method.to_s + ) + + raise SilentModeBlockedError, 'only get, head, options, and trace methods are allowed in silent mode' + end + + def self.blocked_by_silent_mode?(http_method) + ::Gitlab::SilentMode.enabled? && SILENT_MODE_ALLOWED_METHODS.exclude?(http_method) + end end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index fa85d839927..180ccf21264 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -45,27 +45,27 @@ module Gitlab 'bg' => 0, 'cs_CZ' => 0, 'da_DK' => 31, - 'de' => 15, + 'de' => 97, 'en' => 100, 'eo' => 0, - 'es' => 31, + 'es' => 30, 'fil_PH' => 0, 'fr' => 98, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 77, + 'ja' => 98, 'ko' => 18, - 'nb_NO' => 23, + 'nb_NO' => 22, 'nl_NL' => 0, 'pl_PL' => 3, 'pt_BR' => 56, - 'ro_RO' => 84, - 'ru' => 24, + 'ro_RO' => 82, + 'ru' => 23, 'si_LK' => 10, 'tr_TR' => 9, - 'uk' => 54, - 'zh_CN' => 99, + 'uk' => 53, + 'zh_CN' => 98, 'zh_HK' => 1, 'zh_TW' => 99 }.freeze diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 564008e7a73..104c9e6c456 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -85,7 +85,7 @@ module Gitlab end def validate_archive_path - Gitlab::Utils.check_path_traversal!(@archive_path) + Gitlab::PathTraversal.check_path_traversal!(@archive_path) raise(ServiceError, 'Archive path is a symlink') if File.lstat(@archive_path).symlink? raise(ServiceError, 'Archive path is not a file') unless File.file?(@archive_path) diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index e30414265be..c2a1a1f8575 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -56,7 +56,6 @@ excluded_attributes: - :runners_token - :runners_token_encrypted - :saml_discovery_token - - :visibility_level - :trial_ends_on - :shared_runners_minute_limit - :extra_shared_runners_minutes_limit diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index 19d707aaca5..0b92942eb8a 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -65,6 +65,15 @@ module Gitlab # with existing groups name and/or path. group_attributes.delete_attributes('name', 'path') + if @top_level_group.has_parent? + group_attributes.attributes['visibility_level'] = sub_group_visibility_level( + group_attributes.attributes['visibility_level'], + @top_level_group.parent + ) + elsif Gitlab::VisibilityLevel.restricted_level?(group_attributes.attributes['visibility_level']) + group_attributes.delete_attribute('visibility_level') + end + restore_group(@top_level_group, group_attributes) end @@ -86,6 +95,7 @@ module Gitlab parent_id = group_attributes.delete_attribute('parent_id') name = group_attributes.delete_attribute('name') path = group_attributes.delete_attribute('path') + visibility_level = group_attributes.delete_attribute('visibility_level') parent_group = @groups_mapping.fetch(parent_id) { raise(ArgumentError, 'Parent group not found') } @@ -94,7 +104,7 @@ module Gitlab name: name, path: path, parent_id: parent_group.id, - visibility_level: sub_group_visibility_level(group_attributes.attributes, parent_group) + visibility_level: sub_group_visibility_level(visibility_level, parent_group) ).execute group.validate! @@ -124,16 +134,23 @@ module Gitlab end end - def sub_group_visibility_level(group_hash, parent_group) - original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE + def sub_group_visibility_level(visibility_level, parent_group) + parent_visibility_level = parent_group.visibility_level - if parent_group && parent_group.visibility_level < original_visibility_level - Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level) + original_visibility_level = visibility_level || + closest_allowed_level(parent_visibility_level) + + if parent_visibility_level < original_visibility_level + closest_allowed_level(parent_visibility_level) else - original_visibility_level + closest_allowed_level(original_visibility_level) end end + def closest_allowed_level(visibility_level) + Gitlab::VisibilityLevel.closest_allowed_level(visibility_level) + end + def reader strong_memoize(:reader) do Gitlab::ImportExport::Reader.new( diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb deleted file mode 100644 index cf75a2c7fa8..00000000000 --- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class LegacyRelationTreeSaver - include Gitlab::ImportExport::CommandLineUtil - - def serialize(exportable, relations_tree) - Gitlab::ImportExport::FastHashSerializer - .new(exportable, relations_tree) - .execute - end - - def save(tree, dir_path, filename) - mkdir_p(dir_path) - - tree_json = ::JSON.generate(tree) - - File.write(File.join(dir_path, filename), tree_json) - end - end - end -end diff --git a/lib/gitlab/import_export/project/exported_relations_merger.rb b/lib/gitlab/import_export/project/exported_relations_merger.rb index dda3d00d608..b5eba768e56 100644 --- a/lib/gitlab/import_export/project/exported_relations_merger.rb +++ b/lib/gitlab/import_export/project/exported_relations_merger.rb @@ -20,8 +20,8 @@ module Gitlab tar_gz_full_path = File.join(dirpath, filename) decompress_path = File.join(dirpath, relation) - Gitlab::Utils.check_path_traversal!(tar_gz_full_path) - Gitlab::Utils.check_path_traversal!(decompress_path) + Gitlab::PathTraversal.check_path_traversal!(tar_gz_full_path) + Gitlab::PathTraversal.check_path_traversal!(decompress_path) # Download tar.gz download_or_copy_upload( diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 36a3c73271b..410e918649b 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -121,7 +121,6 @@ tree: - label: - :priorities - :service_desk_setting - - :design_management_repository group_members: - :user @@ -322,6 +321,7 @@ included_attributes: - :feature_flags_access_level - :releases_access_level - :infrastructure_access_level + - :model_experiments_access_level prometheus_metrics: - :created_at - :updated_at @@ -742,6 +742,7 @@ included_attributes: - :feature_flags_access_level - :releases_access_level - :infrastructure_access_level + - :model_experiments_access_level - :auto_devops_deploy_strategy - :auto_devops_enabled - :container_registry_enabled diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 5d7e3ea9ed7..8c673acdd1a 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -23,6 +23,7 @@ module Gitlab design: 'DesignManagement::Design', designs: 'DesignManagement::Design', design_management_repository: 'DesignManagement::Repository', + design_management_repository_state: 'Geo::DesignManagementRepositoryState', design_versions: 'DesignManagement::Version', actions: 'DesignManagement::Action', labels: :project_labels, diff --git a/lib/gitlab/import_export/recursive_merge_folders.rb b/lib/gitlab/import_export/recursive_merge_folders.rb index 982358699bd..827385d4daf 100644 --- a/lib/gitlab/import_export/recursive_merge_folders.rb +++ b/lib/gitlab/import_export/recursive_merge_folders.rb @@ -45,9 +45,9 @@ module Gitlab DEFAULT_DIR_MODE = 0o700 def self.merge(source_path, target_path) - Gitlab::Utils.check_path_traversal!(source_path) - Gitlab::Utils.check_path_traversal!(target_path) - Gitlab::Utils.check_allowed_absolute_path!(source_path, [Dir.tmpdir]) + Gitlab::PathTraversal.check_path_traversal!(source_path) + Gitlab::PathTraversal.check_path_traversal!(target_path) + Gitlab::PathTraversal.check_allowed_absolute_path!(source_path, [Dir.tmpdir]) recursive_merge(source_path, target_path) end diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index cc214d730fe..d7d262501de 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -42,7 +42,7 @@ module Gitlab def ensure_repository_does_not_exist! if repository.exists? shared.logger.info( - message: %Q{Deleting existing "#{repository.disk_path}" to re-import it.} + message: %{Deleting existing "#{repository.disk_path}" to re-import it.} ) Repositories::DestroyService.new(repository).execute diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 00a7387afe2..70aaa59f912 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -94,9 +94,9 @@ module Gitlab raise RedisClusterValidator::CrossSlotError, "Redis command #{result[:command_name]} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" end - increment_allowed_cross_slot_request_count if result[:allowed] + increment_allowed_cross_slot_request_count if result[:allowed] && !result[:valid] - result[:valid] + result[:valid] || result[:allowed] end def enable_redis_cluster_validation diff --git a/lib/gitlab/instrumentation/redis_cluster_validator.rb b/lib/gitlab/instrumentation/redis_cluster_validator.rb index 1567e54d8da..948132e6edd 100644 --- a/lib/gitlab/instrumentation/redis_cluster_validator.rb +++ b/lib/gitlab/instrumentation/redis_cluster_validator.rb @@ -193,8 +193,7 @@ module Gitlab keys = commands.map { |command| extract_keys(command) }.flatten { - # calculate key-slots only if not allowed - valid: allow_cross_slot_commands? || !has_cross_slot_keys?(keys), + valid: !has_cross_slot_keys?(keys), command_name: command_name, key_count: keys.size, allowed: allow_cross_slot_commands? @@ -211,6 +210,10 @@ module Gitlab Thread.current[:allow_cross_slot_commands] -= 1 end + def allow_cross_slot_commands? + Thread.current[:allow_cross_slot_commands].to_i > 0 + end + private def extract_keys(command) @@ -226,10 +229,6 @@ module Gitlab keys.map { |key| key_slot(key) }.uniq.many? # rubocop: disable CodeReuse/ActiveRecord end - def allow_cross_slot_commands? - Thread.current[:allow_cross_slot_commands].to_i > 0 - end - def key_slot(key) ::Redis::Cluster::KeySlotConverter.convert(extract_hash_tag(key)) end diff --git a/lib/gitlab/internal_events.rb b/lib/gitlab/internal_events.rb new file mode 100644 index 00000000000..cde83068de1 --- /dev/null +++ b/lib/gitlab/internal_events.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module InternalEvents + class << self + include Gitlab::Tracking::Helpers + + def track_event(event_name, **kwargs) + user_id = kwargs.delete(:user_id) + UsageDataCounters::HLLRedisCounter.track_event(event_name, values: user_id) + + project_id = kwargs.delete(:project_id) + namespace_id = kwargs.delete(:namespace_id) + + namespace = Namespace.find(namespace_id) if namespace_id + + standard_context = Tracking::StandardContext.new( + project_id: project_id, + user_id: user_id, + namespace_id: namespace&.id, + plan_name: namespace&.actual_plan_name + ).to_context + + service_ping_context = Tracking::ServicePingContext.new( + data_source: :redis_hll, + event: event_name + ).to_context + + track_struct_event(event_name, contexts: [standard_context, service_ping_context]) + end + + private + + def track_struct_event(event_name, contexts:) + category = 'InternalEventTracking' + tracker = Gitlab::Tracking.tracker + tracker.event(category, event_name, context: contexts) + rescue StandardError => error + Gitlab::ErrorTracking + .track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: event_name) + end + end + end +end diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb deleted file mode 100644 index 0087c2accc3..00000000000 --- a/lib/gitlab/json_cache.rb +++ /dev/null @@ -1,118 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class JsonCache - attr_reader :backend, :namespace - - STRATEGY_KEY_COMPONENTS = { - revision: Gitlab.revision, - version: [Gitlab::VERSION, Rails.version] - }.freeze - - def initialize(options = {}) - @backend = options.fetch(:backend, Rails.cache) - @namespace = options.fetch(:namespace, nil) - @cache_key_strategy = options.fetch(:cache_key_strategy, :revision) - end - - def active? - if backend.respond_to?(:active?) - backend.active? - else - true - end - end - - def cache_key(key) - expanded_cache_key = [namespace, key, *strategy_key_component].compact - expanded_cache_key.join(':').freeze - end - - def strategy_key_component - STRATEGY_KEY_COMPONENTS.fetch(@cache_key_strategy) - end - - def expire(key) - backend.delete(cache_key(key)) - end - - def read(key, klass = nil) - value = backend.read(cache_key(key)) - value = parse_value(value, klass) unless value.nil? - value - end - - def write(key, value, options = nil) - backend.write(cache_key(key), value.to_json, options) - end - - def fetch(key, options = {}, &block) - klass = options.delete(:as) - value = read(key, klass) - - return value unless value.nil? - - value = yield - - write(key, value, options) - - value - end - - private - - def parse_value(raw, klass) - value = Gitlab::Json.parse(raw.to_s) - - case value - when Hash then parse_entry(value, klass) - when Array then parse_entries(value, klass) - else - value - end - rescue JSON::ParserError - nil - end - - def parse_entry(raw, klass) - return unless valid_entry?(raw, klass) - return klass.new(raw) unless klass.ancestors.include?(ActiveRecord::Base) - - # When the cached value is a persisted instance of ActiveRecord::Base in - # some cases a relation can return an empty collection becauses scope.none! - # is being applied on ActiveRecord::Associations::CollectionAssociation#scope - # when the new_record? method incorrectly returns false. - # - # See https://gitlab.com/gitlab-org/gitlab/issues/9903#note_145329964 - klass.allocate.init_with(encode_for(klass, raw)) - end - - def encode_for(klass, raw) - # We have models that leave out some fields from the JSON export for - # security reasons, e.g. models that include the CacheMarkdownField. - # The ActiveRecord::AttributeSet we build from raw does know about - # these columns so we need manually set them. - missing_attributes = (klass.columns.map(&:name) - raw.keys) - missing_attributes.each { |column| raw[column] = nil } - - coder = {} - klass.new(raw).encode_with(coder) - coder["new_record"] = new_record?(raw, klass) - coder - end - - def new_record?(raw, klass) - raw.fetch(klass.primary_key, nil).blank? - end - - def valid_entry?(raw, klass) - return false unless klass && raw.is_a?(Hash) - - (raw.keys - klass.attribute_names).empty? - end - - def parse_entries(values, klass) - values.filter_map { |value| parse_entry(value, klass) } - end - end -end diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb index 8cab069e1bf..f742cb82b8d 100644 --- a/lib/gitlab/markdown_cache/redis/store.rb +++ b/lib/gitlab/markdown_cache/redis/store.rb @@ -11,7 +11,7 @@ module Gitlab Gitlab::Redis::Cache.with do |r| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - r.pipelined do |pipeline| + Gitlab::Redis::CrossSlot::Pipeline.new(r).pipelined do |pipeline| subjects.each do |subject| results[subject.cache_key] = new(subject).read(pipeline) end diff --git a/lib/gitlab/merge_requests/message_generator.rb b/lib/gitlab/merge_requests/message_generator.rb index 5113fbdcd7b..523e0e665dc 100644 --- a/lib/gitlab/merge_requests/message_generator.rb +++ b/lib/gitlab/merge_requests/message_generator.rb @@ -51,6 +51,7 @@ module Gitlab end, 'description' => ->(merge_request, _, _) { merge_request.description }, 'reference' => ->(merge_request, _, _) { merge_request.to_reference(full: true) }, + 'local_reference' => ->(merge_request, _, _) { merge_request.to_reference(full: false) }, 'first_commit' => -> (merge_request, _, _) { return unless merge_request.persisted? || merge_request.compare_commits.present? diff --git a/lib/gitlab/metrics/loose_foreign_keys_slis.rb b/lib/gitlab/metrics/loose_foreign_keys_slis.rb index 5d8245aa609..c0db709fe13 100644 --- a/lib/gitlab/metrics/loose_foreign_keys_slis.rb +++ b/lib/gitlab/metrics/loose_foreign_keys_slis.rb @@ -26,7 +26,7 @@ module Gitlab private def possible_labels - ::Gitlab::Database.db_config_names.map do |db_config_name| + ::Gitlab::Database.db_config_names(with_schema: :gitlab_shared).map do |db_config_name| { db_config_name: db_config_name, feature_category: :database diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index b4e9e85a012..e2cdd6c5358 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -13,7 +13,7 @@ module Gitlab return unless current_transaction - labels = { store: event.payload[:store].split('::').last } + labels = { store: extract_store_name(event) } current_transaction.observe(:gitlab_cache_read_multikey_count, event.payload[:key].size, labels) do buckets [10, 50, 100, 1000] docstring 'Number of keys for mget in read_multi/fetch_multi' @@ -26,11 +26,7 @@ module Gitlab return unless current_transaction return if event.payload[:super_operation] == :fetch - unless event.payload[:hit] - current_transaction.increment(:gitlab_cache_misses_total, 1) do - docstring 'Cache read miss' - end - end + track_cache_miss(event) unless event.payload[:hit] end def cache_write(event) @@ -48,23 +44,23 @@ module Gitlab def cache_fetch_hit(event) return unless current_transaction - current_transaction.increment(:gitlab_transaction_cache_read_hit_count_total, 1) + labels = { store: extract_store_name(event) } + current_transaction.increment(:gitlab_transaction_cache_read_hit_count_total, 1, labels) end def cache_generate(event) return unless current_transaction - current_transaction.increment(:gitlab_cache_misses_total, 1) do - docstring 'Cache read miss' - end + track_cache_miss(event) - current_transaction.increment(:gitlab_transaction_cache_read_miss_count_total, 1) + labels = { store: extract_store_name(event) } + current_transaction.increment(:gitlab_transaction_cache_read_miss_count_total, 1, labels) end def observe(key, event) return unless current_transaction - labels = { operation: key, store: event.payload[:store].split('::').last } + labels = { operation: key, store: extract_store_name(event) } current_transaction.increment(:gitlab_cache_operations_total, 1, labels) do docstring 'Cache operations' @@ -76,6 +72,20 @@ module Gitlab private + def track_cache_miss(event) + # avoid passing in labels to ensure metric has consistent set of labels + labels = { store: extract_store_name(event) } + + current_transaction.increment(:gitlab_cache_misses_total, 1, labels) do + docstring 'Cache read miss' + end + end + + def extract_store_name(event) + # see payload documentation in https://guides.rubyonrails.org/active_support_instrumentation.html#active-support + event.payload[:store].to_s.split('::').last + end + def current_transaction ::Gitlab::Metrics::WebTransaction.current end diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb index cc485d8a5db..1f15f1d5857 100644 --- a/lib/gitlab/middleware/compressed_json.rb +++ b/lib/gitlab/middleware/compressed_json.rb @@ -3,7 +3,6 @@ module Gitlab module Middleware class CompressedJson - COLLECTOR_PATH = '/api/v4/error_tracking/collector' INSTANCE_PACKAGES_PATH = %r{ \A/api/v4/packages/npm/-/npm/v1/security/ (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) @@ -79,8 +78,7 @@ module Gitlab end def match_path?(env) - env['PATH_INFO'].start_with?((File.join(relative_url, COLLECTOR_PATH))) || - match_packages_path?(env) + match_packages_path?(env) end def match_packages_path?(env) diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index a21d0228082..ee8259cc671 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -6,7 +6,8 @@ module Gitlab SUPPORTED_ORDERING = { Group => { name: :asc }, AuditEvent => { id: :desc }, - ::Ci::Build => { id: :desc } + ::Ci::Build => { id: :desc }, + ::Packages::BuildInfo => { id: :desc } }.freeze # Relation types that are enforced in this list diff --git a/lib/gitlab/patch/redis_cache_store.rb b/lib/gitlab/patch/redis_cache_store.rb new file mode 100644 index 00000000000..5279c4081b2 --- /dev/null +++ b/lib/gitlab/patch/redis_cache_store.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Gitlab + module Patch + module RedisCacheStore + PIPELINE_BATCH_SIZE = 100 + + # We will try keep patched code explicit and matching the original signature in + # https://github.com/rails/rails/blob/v6.1.7.2/activesupport/lib/active_support/cache/redis_cache_store.rb#L361 + def read_multi_mget(*names) # rubocop:disable Style/ArgumentsForwarding + return super unless enable_rails_cache_pipeline_patch? + return super unless use_patched_mget? + + patched_read_multi_mget(*names) # rubocop:disable Style/ArgumentsForwarding + end + + # `delete_multi_entries` in Rails runs a multi-key `del` command + # patch will run pipelined single-key `del` for Redis Cluster compatibility + def delete_multi_entries(entries, **options) + return super unless enable_rails_cache_pipeline_patch? + + delete_count = 0 + redis.with do |conn| + entries.each_slice(PIPELINE_BATCH_SIZE) do |subset| + delete_count += Gitlab::Redis::CrossSlot::Pipeline.new(conn).pipelined do |pipeline| + subset.each { |entry| pipeline.del(entry) } + end.sum + end + end + delete_count + end + + # Copied from https://github.com/rails/rails/blob/v6.1.6.1/activesupport/lib/active_support/cache/redis_cache_store.rb + # re-implements `read_multi_mget` using a pipeline of `get`s rather than an `mget` + # + def patched_read_multi_mget(*names) + options = names.extract_options! + options = merged_options(options) + return {} if names == [] + + raw = options&.fetch(:raw, false) + + keys = names.map { |name| normalize_key(name, options) } + + values = failsafe(:patched_read_multi_mget, returning: {}) do + redis.with { |c| pipeline_mget(c, keys) } + end + + names.zip(values).each_with_object({}) do |(name, value), results| + if value # rubocop:disable Style/Next + entry = deserialize_entry(value, raw: raw) + unless entry.nil? || entry.expired? || entry.mismatched?(normalize_version(name, options)) + results[name] = entry.value + end + end + end + end + + def pipeline_mget(conn, keys) + keys.each_slice(PIPELINE_BATCH_SIZE).flat_map do |subset| + Gitlab::Redis::CrossSlot::Pipeline.new(conn).pipelined do |p| + subset.each { |key| p.get(key) } + end + end + end + + private + + def enable_rails_cache_pipeline_patch? + redis.with { |c| ::Gitlab::Redis::ClusterUtil.cluster?(c) } + end + + # MultiStore reads ONLY from the default store (no fallback), hence we can use `mget` + # if the default store is not a Redis::Cluster. We should do that as pipelining gets on a single redis is slow + def use_patched_mget? + redis.with do |conn| + next true unless conn.is_a?(Gitlab::Redis::MultiStore) + + ::Gitlab::Redis::ClusterUtil.cluster?(conn.default_store) + end + end + end + end +end diff --git a/lib/gitlab/patch/redis_cluster.rb b/lib/gitlab/patch/redis_cluster.rb new file mode 100644 index 00000000000..145ce35a317 --- /dev/null +++ b/lib/gitlab/patch/redis_cluster.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# Patch to expose `find_node_key` method for cross-slot pipelining +# In redis v5.0.x, cross-slot pipelining is implemented via redis-cluster-client. +# This patch should be removed since there is no need for it. +# Gitlab::Redis::CrossSlot and its usage should be removed as well. +if Gem::Version.new(Redis::VERSION) != Gem::Version.new('4.8.0') + raise 'New version of redis detected, please remove or update this patch' +end + +module Gitlab + module Patch + module RedisCluster + # _find_node_key exposes a private function of the same name in Redis::Cluster. + # See https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis/cluster.rb#L282 + def _find_node_key(command) + find_node_key(command) + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index b0804c2ff66..e112423f167 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -122,6 +122,7 @@ module Gitlab ILLEGAL_PROJECT_PATH_WORDS = PROJECT_WILDCARD_ROUTES ILLEGAL_GROUP_PATH_WORDS = (PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze + ILLEGAL_ORGANIZATION_PATH_WORDS = (TOP_LEVEL_ROUTES | PROJECT_WILDCARD_ROUTES | GROUP_ROUTES).freeze # The namespace regex is used in JavaScript to validate usernames in the "Register" form. However, Javascript # does not support the negative lookbehind assertion (?<!) that disallows usernames ending in `.git` and `.atom`. @@ -138,6 +139,17 @@ module Gitlab PROJECT_PATH_FORMAT_REGEX = /(?:#{PATH_REGEX_STR})#{NO_SUFFIX_REGEX}/.freeze FULL_NAMESPACE_FORMAT_REGEX = %r{(#{NAMESPACE_FORMAT_REGEX}/){,#{Namespace::NUMBER_OF_ANCESTORS_ALLOWED}}#{NAMESPACE_FORMAT_REGEX}}.freeze + def organization_route_regex + @organization_route_regex ||= begin + illegal_words = Regexp.new(Regexp.union(ILLEGAL_ORGANIZATION_PATH_WORDS).source, Regexp::IGNORECASE) + + single_line_regexp %r{ + (?!(#{illegal_words})/) + #{NAMESPACE_FORMAT_REGEX} + }x + end + end + def root_namespace_route_regex @root_namespace_route_regex ||= begin illegal_words = Regexp.new(Regexp.union(TOP_LEVEL_ROUTES).source, Regexp::IGNORECASE) @@ -195,6 +207,10 @@ module Gitlab @full_namespace_path_regex ||= %r{\A#{full_namespace_route_regex}/\z} end + def organization_path_regex + @organization_path_regex ||= %r{\A#{organization_route_regex}/\z} + end + def full_project_path_regex @full_project_path_regex ||= %r{\A#{full_namespace_route_regex}/#{project_route_regex}/\z} end diff --git a/lib/gitlab/path_traversal.rb b/lib/gitlab/path_traversal.rb new file mode 100644 index 00000000000..1123ff73136 --- /dev/null +++ b/lib/gitlab/path_traversal.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module PathTraversal + extend self + PathTraversalAttackError = Class.new(StandardError) + + private_class_method def logger + @logger ||= Gitlab::AppLogger + end + + PATH_TRAVERSAL_REGEX = %r{(\A(\.{1,2})\z|\A\.\.[/\\]|[/\\]\.\.\z|[/\\]\.\.[/\\]|\n)} + + # Ensure that the relative path will not traverse outside the base directory + # We url decode the path to avoid passing invalid paths forward in url encoded format. + # Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580 + # It also checks for ALT_SEPARATOR aka '\' (forward slash) + def check_path_traversal!(path) + return unless path + + path = path.to_s if path.is_a?(Gitlab::HashedPath) + raise PathTraversalAttackError, 'Invalid path' unless path.is_a?(String) + + path = ::Gitlab::Utils.decode_path(path) + + if path.match?(PATH_TRAVERSAL_REGEX) + logger.warn(message: "Potential path traversal attempt detected", path: path.to_s) + raise PathTraversalAttackError, 'Invalid path' + end + + path + end + + def check_allowed_absolute_path!(path, allowlist) + return unless Pathname.new(path).absolute? + return if ::Gitlab::Utils.allowlisted?(path, allowlist) + + raise StandardError, "path #{path} is not allowed" + end + + def check_allowed_absolute_path_and_path_traversal!(path, path_allowlist) + traversal_path = check_path_traversal!(path) + raise StandardError, "path is not a string!" unless traversal_path.is_a?(String) + + check_allowed_absolute_path!(traversal_path, path_allowlist) + end + end +end diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb index c770260a66e..0fcb8321dae 100644 --- a/lib/gitlab/project_authorizations.rb +++ b/lib/gitlab/project_authorizations.rb @@ -12,62 +12,56 @@ module Gitlab end def calculate - cte = if Feature.enabled?(:linear_project_authorization, user) - linear_cte - else - recursive_cte - end - - cte_alias = cte.table.alias(Group.table_name) - projects = Project.arel_table - links = ProjectGroupLink.arel_table - - relations = [ - # The project a user has direct access to. - user.projects_with_active_memberships.select_for_project_authorization, - - # The personal projects of the user. - user.personal_projects.select_project_owner_for_project_authorization, + if Feature.enabled?(:compare_project_authorization_linear_cte, user) + linear_relation = calculate_with_linear_query + recursive_relation = calculate_with_recursive_query + recursive_set = Set.new(recursive_relation.to_a.pluck(:project_id, :access_level)) + linear_set = Set.new(linear_relation.to_a.pluck(:project_id, :access_level)) + if linear_set == recursive_set + Gitlab::AppJsonLogger.info(event: 'linear_authorized_projects_check', + user_id: user.id, + matching_results: true) + return calculate_with_linear_query + else + Gitlab::AppJsonLogger.warn(event: 'linear_authorized_projects_check', + user_id: user.id, + matching_results: false) + end + end - # Projects that belong directly to any of the groups the user has - # access to. - Namespace - .unscoped - .select([alias_as_column(projects[:id], 'project_id'), - cte_alias[:access_level]]) - .from(cte_alias) - .joins(:projects), - - # Projects shared with any of the namespaces the user has access to. - Namespace - .unscoped - .select([ - links[:project_id], - least(cte_alias[:access_level], links[:group_access], 'access_level') - ]) - .from(cte_alias) - .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') - .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') - .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') - .where('p_ns.share_with_group_lock IS FALSE') - ] + Gitlab::AppJsonLogger.info(event: 'linear_authorized_projects_check_with_flag', + feature_flag_status: Feature.enabled?(:linear_project_authorization, user)) if Feature.enabled?(:linear_project_authorization, user) - ProjectAuthorization - .unscoped - .with(cte.to_arel) - .select_from_union(relations) + calculate_with_linear_query else - ProjectAuthorization - .unscoped - .with - .recursive(cte.to_arel) - .select_from_union(relations) + calculate_with_recursive_query end end private + def calculate_with_linear_query + cte = linear_cte + cte_alias = cte.table.alias(Group.table_name) + + ProjectAuthorization + .unscoped + .with(cte.to_arel) + .select_from_union(relations(cte_alias: cte_alias)) + end + + def calculate_with_recursive_query + cte = recursive_cte + cte_alias = cte.table.alias(Group.table_name) + + ProjectAuthorization + .unscoped + .with + .recursive(cte.to_arel) + .select_from_union(relations(cte_alias: cte_alias)) + end + # Builds a recursive CTE that gets all the groups the current user has # access to, including any nested groups and any shared groups. def recursive_cte @@ -83,9 +77,11 @@ module Gitlab # Namespaces shared with any of the group cte << Group.select([namespaces[:id], - least(members[:access_level], - group_group_links[:group_access], - 'access_level')]) + least( + members[:access_level], + group_group_links[:group_access], + 'access_level' + )]) .joins(join_group_group_links) .joins(join_members_on_group_group_links) @@ -120,7 +116,9 @@ module Gitlab .merge(user.group_members) .merge(GroupMember.active_state) - union = Namespace.from_union([shared_groups, member_groups_with_ancestors]) + union = Namespace + .select("namespaces.id, access_level") + .from_union([shared_groups, member_groups_with_ancestors]) Gitlab::SQL::CTE.new(:linear_namespaces_cte, union) end @@ -185,5 +183,45 @@ module Gitlab def alias_as_column(value, alias_to) Arel::Nodes::As.new(value, Arel::Nodes::SqlLiteral.new(alias_to)) end + + def relations(cte_alias:) + [ + user.projects_with_active_memberships.select_for_project_authorization, + user.personal_projects.select_project_owner_for_project_authorization, + projects_belonging_directy_to_any_groups_user_has_access_to(cte_alias: cte_alias), + projects_shared_with_namespaces_user_has_access_to(cte_alias: cte_alias) + ] + end + + def projects_shared_with_namespaces_user_has_access_to(cte_alias:) + Namespace + .unscoped + .select([ + links[:project_id], + least(cte_alias[:access_level], links[:group_access], 'access_level') + ]) + .from(cte_alias) + .joins('INNER JOIN project_group_links ON project_group_links.group_id = namespaces.id') + .joins('INNER JOIN projects ON projects.id = project_group_links.project_id') + .joins('INNER JOIN namespaces p_ns ON p_ns.id = projects.namespace_id') + .where('p_ns.share_with_group_lock IS FALSE') + end + + def projects_belonging_directy_to_any_groups_user_has_access_to(cte_alias:) + Namespace + .unscoped + .select([alias_as_column(projects[:id], 'project_id'), + cte_alias[:access_level]]) + .from(cte_alias) + .joins(:projects) + end + + def projects + Project.arel_table + end + + def links + ProjectGroupLink.arel_table + end end end diff --git a/lib/gitlab/quick_actions/relate_actions.rb b/lib/gitlab/quick_actions/relate_actions.rb index b8cbfdefda1..058c1e7e9bf 100644 --- a/lib/gitlab/quick_actions/relate_actions.rb +++ b/lib/gitlab/quick_actions/relate_actions.rb @@ -16,32 +16,60 @@ module Gitlab end params '<#issue | group/project#issue | issue URL>' types Issue - condition { can_relate_issues? } + condition { can_admin_link? } parse_params { |issues| format_params(issues) } command :relate do |target_issues| create_links(target_issues) end + desc { _("Remove link with another issue") } + explanation do |issue| + _('Removes link with %{issue_ref}.') % { issue_ref: issue.to_reference(quick_action_target) } + end + execution_message do |issue| + _('Removed link with %{issue_ref}.') % { issue_ref: issue.to_reference(quick_action_target) } + end + params '<#issue | group/project#issue | issue URL>' + types Issue + condition { can_admin_link? } + parse_params do |issue_param| + extract_references(issue_param, :issue).first + end + command :unlink do |issue| + link = IssueLink.for_issues(quick_action_target, issue).first + + if link + call_link_service(IssueLinks::DestroyService.new(link, current_user)) + else + @execution_message[:unlink] = _('No linked issue matches the provided parameter.') + end + end + private + def can_admin_link? + current_user.can?(:admin_issue_link, quick_action_target) + end + def create_links(references, type: 'relates_to') - service = IssueLinks::CreateService.new( + create_service_instance = IssueLinks::CreateService.new( quick_action_target, current_user, { issuable_references: references, link_type: type } ) - create_issue_link = proc { service.execute } + + call_link_service(create_service_instance) + end + + def call_link_service(service_instance) + execute_service = proc { service_instance.execute } if quick_action_target.persisted? - create_issue_link.call + execute_service.call else - quick_action_target.run_after_commit(&create_issue_link) + quick_action_target.run_after_commit(&execute_service) end end - def can_relate_issues? - current_user.can?(:admin_issue_link, quick_action_target) - end - def format_params(issue_references) issue_references.split(' ') end diff --git a/lib/gitlab/quick_actions/work_item_actions.rb b/lib/gitlab/quick_actions/work_item_actions.rb index fa43308c9e2..5664410f3ca 100644 --- a/lib/gitlab/quick_actions/work_item_actions.rb +++ b/lib/gitlab/quick_actions/work_item_actions.rb @@ -12,42 +12,94 @@ module Gitlab format(_("Converts work item to %{type}. Widgets not supported in new type are removed."), type: target_type) end types WorkItem - condition do - quick_action_target&.project&.work_items_mvc_2_feature_flag_enabled? - end params 'Task | Objective | Key Result | Issue' command :type do |type_name| - work_item_type = ::WorkItems::Type.find_by_name(type_name) - errors = validate_type(work_item_type) - - if errors.present? - @execution_message[:type] = errors - else - @updates[:issue_type] = work_item_type.base_type - @updates[:work_item_type] = work_item_type - @execution_message[:type] = _('Type changed successfully.') - end + @execution_message[:type] = update_type(type_name, :type) + end + + desc { _('Promote work item') } + explanation do |type_name| + format(_("Promotes work item to %{type}."), type: type_name) + end + types WorkItem + params 'issue | objective' + condition { supports_promotion? } + command :promote_to do |type_name| + @execution_message[:promote_to] = update_type(type_name, :promote_to) end end private + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def update_type(type_name, command) + new_type = ::WorkItems::Type.find_by_name(type_name.titleize) + error_message = command == :type ? validate_type(new_type) : validate_promote_to(new_type) + return error_message if error_message.present? + + @updates[:issue_type] = new_type.base_type + @updates[:work_item_type] = new_type + + success_msg[command] + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + def validate_type(type) - return type_error(:not_found) unless type.present? - return type_error(:same_type) if quick_action_target.work_item_type == type - return type_error(:forbidden) unless current_user.can?(:"create_#{type.base_type}", quick_action_target) + return error_msg(:not_found) unless type.present? + return error_msg(:same_type) if quick_action_target.work_item_type == type + return error_msg(:forbidden) unless current_user.can?(:"create_#{type.base_type}", quick_action_target) nil end - def type_error(reason) + def validate_promote_to(type) + return error_msg(:not_found, action: 'promote') unless type && supports_promote_to?(type.name) + + unless current_user.can?(:"create_#{type.base_type}", quick_action_target) + return error_msg(:forbidden, action: 'promote') + end + + validate_hierarchy + end + + def validate_hierarchy + return unless current_type.task? && quick_action_target.parent_link + + error_msg(:hierarchy, action: 'promote') + end + + def current_type + quick_action_target.work_item_type + end + + def supports_promotion? + current_type.base_type.in?(promote_to_map.keys) + end + + def supports_promote_to?(type_name) + type_name == promote_to_map[current_type.base_type] + end + + def promote_to_map + { issue: 'Incident', task: 'Issue' }.with_indifferent_access + end + + def error_msg(reason, action: 'convert') message = { not_found: 'Provided type is not supported', same_type: 'Types are the same', - forbidden: 'You have insufficient permissions' + forbidden: 'You have insufficient permissions', + hierarchy: 'A task cannot be promoted when a parent issue is present' }.freeze - format(_("Failed to convert this work item: %{reason}."), { reason: message[reason] }) + format(_("Failed to %{action} this work item: %{reason}."), { action: action, reason: message[reason] }) + end + + def success_msg + { + type: _('Type changed successfully.'), + promote_to: _("Work Item promoted successfully.") + } end end end diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index dc13bb927e6..9f5769a5c97 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -11,17 +11,15 @@ module Gitlab end def clear_cache!(key) - use_pipeline = ::Feature.enabled?(:use_pipeline_over_multikey) - with do |redis| keys = read(key).map { |value| "#{cache_namespace}:#{value}" } keys << cache_key(key) Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |pipeline| - if use_pipeline - keys.each { |key| pipeline.unlink(key) } - else + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::ClusterUtil.batch_unlink(keys, redis) + else + redis.pipelined do |pipeline| keys.each_slice(1000) { |subset| pipeline.unlink(*subset) } end end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 64ca89c6bff..06bce7649bf 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -9,6 +9,7 @@ module Gitlab # config/initializers/7_redis.rb, instrumented, and used in health- & readiness checks. ALL_CLASSES = [ Gitlab::Redis::Cache, + Gitlab::Redis::ClusterCache, Gitlab::Redis::DbLoadBalancing, Gitlab::Redis::FeatureFlag, Gitlab::Redis::Queues, @@ -16,7 +17,8 @@ module Gitlab Gitlab::Redis::RepositoryCache, Gitlab::Redis::Sessions, Gitlab::Redis::SharedState, - Gitlab::Redis::TraceChunks + Gitlab::Redis::TraceChunks, + Gitlab::Redis::Chat ].freeze end end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index ba3af3e7a6f..60944268f91 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -5,19 +5,35 @@ module Gitlab class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab' - # Full list of options: - # https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html#method-c-new - def self.active_support_config - { - redis: pool, - compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), - namespace: CACHE_NAMESPACE, - expires_in: default_ttl_seconds - } - end + class << self + # Full list of options: + # https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html#method-c-new + def active_support_config + { + redis: pool, + compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), + namespace: CACHE_NAMESPACE, + expires_in: default_ttl_seconds + } + end + + def default_ttl_seconds + ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i + end + + # Exposes redis for Peek adapter. To be removed after ClusterCache migration. + def multistore_redis + redis + end + + private + + def redis + primary_store = ::Redis.new(Gitlab::Redis::ClusterCache.params) + secondary_store = ::Redis.new(params) - def self.default_ttl_seconds - ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i + MultiStore.new(primary_store, secondary_store, store_name) + end end end end diff --git a/lib/gitlab/redis/chat.rb b/lib/gitlab/redis/chat.rb new file mode 100644 index 00000000000..6f320fa6fc6 --- /dev/null +++ b/lib/gitlab/redis/chat.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class Chat < ::Gitlab::Redis::Wrapper + class << self + def config_fallback + Cache + end + end + end + end +end diff --git a/lib/gitlab/redis/cluster_cache.rb b/lib/gitlab/redis/cluster_cache.rb new file mode 100644 index 00000000000..15a87739c6d --- /dev/null +++ b/lib/gitlab/redis/cluster_cache.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class ClusterCache < ::Gitlab::Redis::Wrapper + class << self + def config_fallback + Cache + end + end + end + end +end diff --git a/lib/gitlab/redis/cluster_util.rb b/lib/gitlab/redis/cluster_util.rb new file mode 100644 index 00000000000..5f1f39b5237 --- /dev/null +++ b/lib/gitlab/redis/cluster_util.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + module ClusterUtil + class << self + # clusters? is used to select Redis command types, on `true`, the subsequent + # commands should be compatible with Redis Cluster. + # + # When working with MultiStore, if even 1 of 2 stores is a Redis::Cluster, + # we should err on the side of caution and return `true `, + def cluster?(obj) + if obj.is_a?(MultiStore) + cluster?(obj.primary_store) || cluster?(obj.secondary_store) + else + obj.respond_to?(:_client) && obj._client.is_a?(::Redis::Cluster) + end + end + + def batch_unlink(keys, redis) + expired_count = 0 + keys.each_slice(1000) do |subset| + expired_count += Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| + subset.each { |key| pipeline.unlink(key) } + end.sum + end + expired_count + end + end + end + end +end diff --git a/lib/gitlab/redis/cross_slot.rb b/lib/gitlab/redis/cross_slot.rb new file mode 100644 index 00000000000..e5aa6d9ce72 --- /dev/null +++ b/lib/gitlab/redis/cross_slot.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + module CrossSlot + class Router + attr_reader :node_mapping, :futures, :node_sequence, :cmd_queue + + delegate :respond_to_missing?, to: :@redis + + # This map contains redis-rb methods which does not map directly + # to a standard Redis command. It is used transform unsupported commands to standard commands + # to find the node key for unsupported commands. + # + # Redis::Cluster::Command only contains details of commands which the Redis Server + # returns. Hence, commands like mapped_hmget and hscan_each internally will call the + # base command, hmget and hscan respectively. + # + # See https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis/cluster/command.rb + UNSUPPORTED_CMD_MAPPING = { + # Internally, redis-rb calls the supported Redis command and transforms the output. + # See https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis/commands/hashes.rb#L104 + mapped_hmget: :hmget + }.freeze + + # Initializes the CrossSlot::Router + # @param {::Redis} + def initialize(redis) + @redis = redis + @node_mapping = {} + @futures = {} + @node_sequence = [] + @cmd_queue = [] + end + + # For now we intercept every redis.call and return a Gitlab-Future object. + # This method groups every commands to a node for fan-out. Commands are grouped using the first key. + # + # rubocop:disable Style/MissingRespondToMissing + def method_missing(cmd, *args, **kwargs, &blk) + # Note that we can re-map the command without affecting execution as it is + # solely for finding the node key. The original cmd will be executed. + node = @redis._client._find_node_key([UNSUPPORTED_CMD_MAPPING.fetch(cmd, cmd)] + args) + + @node_mapping[node] ||= [] + @futures[node] ||= [] + + @node_sequence << node + @node_mapping[node] << [cmd, args, kwargs || {}, blk] + f = Future.new + @futures[node] << f + @cmd_queue << [f, cmd, args, kwargs || {}, blk] + f + end + # rubocop:enable Style/MissingRespondToMissing + end + + # Wraps over redis-rb's Future in + # https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis/pipeline.rb#L244 + class Future + def set(future, is_val = false) + @redis_future = future + @is_val = is_val + end + + def value + return @redis_val if @is_val + + @redis_future.value + end + end + + # Pipeline allows cross-slot pipelined to be called. The fan-out logic is implemented in + # https://github.com/redis-rb/redis-cluster-client/blob/master/lib/redis_client/cluster/pipeline.rb + # which is available in redis-rb v5.0. + # + # This file can be deprecated after redis-rb v4.8.0 is upgraded to v5.0 + class Pipeline + # Initializes the CrossSlot::Pipeline + # @param {::Redis} + def initialize(redis) + @redis = redis + end + + # pipelined is used in place of ::Redis `.pipelined` when running in a cluster context + # where cross-slot operations may happen. + def pipelined(&block) + # Directly call .pipelined and defer the pipeline execution to MultiStore. + # MultiStore could wrap over 0, 1, or 2 Redis Cluster clients, handling it here + # will not work for 2 clients since the key-slot topology can differ. + if use_cross_slot_pipelining? + router = Router.new(@redis) + yield router + execute_commands(router) + else + # use redis-rb's pipelined method + @redis.pipelined(&block) + end + end + + private + + def use_cross_slot_pipelining? + !@redis.instance_of?(::Gitlab::Redis::MultiStore) && @redis._client.instance_of?(::Redis::Cluster) + end + + def execute_commands(router) + router.node_mapping.each do |node_key, commands| + # TODO possibly use Threads to speed up but for now `n` is 3-5 which is small. + @redis.pipelined do |p| + commands.each_with_index do |command, idx| + future = router.futures[node_key][idx] + cmd, args, kwargs, blk = command + future.set(p.public_send(cmd, *args, **kwargs, &blk)) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + + router.node_sequence.map do |node_key| + router.futures[node_key].shift.value + end + rescue ::Redis::CommandError => err + if err.message.start_with?('MOVED', 'ASK') + Gitlab::ErrorTracking.log_exception(err) + return execute_commands_sequentially(router) + end + + raise + end + + def execute_commands_sequentially(router) + router.cmd_queue.map do |command| + future, cmd, args, kwargs, blk = command + future.set(@redis.public_send(cmd, *args, **kwargs, &blk), true) # rubocop:disable GitlabSecurity/PublicSend + future.value + end + end + end + end + end +end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index 9571e2f92e6..d36ef6b99ee 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -44,6 +44,7 @@ module Gitlab hscan_each mapped_hmget mget + scan scan_each scard sismember @@ -66,11 +67,14 @@ module Gitlab mapped_hmset rpush sadd + sadd? set setex setnx srem unlink + + memory ].freeze PIPELINED_COMMANDS = %i[ @@ -122,7 +126,7 @@ module Gitlab if use_primary_and_secondary_stores? pipelined_both(name, *args, **kwargs, &block) else - default_store.send(name, *args, **kwargs, &block) + send_command(default_store, name, *args, **kwargs, &block) end end end @@ -289,6 +293,16 @@ module Gitlab # rubocop:disable GitlabSecurity/PublicSend def send_command(redis_instance, command_name, *args, **kwargs, &block) + # Run wrapped pipeline for each instance individually so that the fan-out is distinct. + # If both primary and secondary are Redis Clusters, the slot-node distribution could + # be different. + # + # We ignore args and kwargs since `pipelined` does not accept arguments + # See https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis.rb#L164 + if command_name.to_s == 'pipelined' && redis_instance._client.instance_of?(::Redis::Cluster) + return Gitlab::Redis::CrossSlot::Pipeline.new(redis_instance).pipelined(&block) + end + if block # Make sure that block is wrapped and executed only on the redis instance that is executing the block redis_instance.send(command_name, *args, **kwargs) do |*params| diff --git a/lib/gitlab/redis/rate_limiting.rb b/lib/gitlab/redis/rate_limiting.rb index 74b4ca12d18..30ec44b748d 100644 --- a/lib/gitlab/redis/rate_limiting.rb +++ b/lib/gitlab/redis/rate_limiting.rb @@ -3,18 +3,11 @@ module Gitlab module Redis class RateLimiting < ::Gitlab::Redis::Wrapper - # We create a subclass only for the purpose of differentiating between different stores in cache metrics - RateLimitingStore = Class.new(ActiveSupport::Cache::RedisCacheStore) - class << self # The data we store on RateLimiting used to be stored on Cache. def config_fallback Cache end - - def cache_store - @cache_store ||= RateLimitingStore.new(redis: pool, namespace: Cache::CACHE_NAMESPACE) - end end end end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index eb99805e2e8..26ca9d2547c 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -548,8 +548,8 @@ module Gitlab # https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html # Avoids linking CVE IDs (https://cve.mitre.org/cve/identifiers/syntaxchange.html#new) as Jira issues. # CVE IDs use the format of CVE-YYYY-NNNNNNN - def jira_issue_key_regex - @jira_issue_key_regex ||= /(?!CVE-\d+-\d+)[A-Z][A-Z_0-9]+-\d+/ + def jira_issue_key_regex(expression_escape: '\b') + /#{expression_escape}(?!CVE-\d+-\d+)[A-Z][A-Z_0-9]+-\d+/ end def jira_issue_key_project_key_extraction_regex @@ -623,6 +623,18 @@ module Gitlab def x509_subject_key_identifier_regex @x509_subject_key_identifier_regex ||= /\A(?:\h{2}:)*\h{2}\z/.freeze end + + def ml_model_version_regex + maven_version_regex + end + + def ml_model_name_regex + package_name_regex + end + + def ml_model_file_name_regex + maven_file_name_regex + end end end diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb index 1f3c084e194..fab0e9e09e8 100644 --- a/lib/gitlab/repository_hash_cache.rb +++ b/lib/gitlab/repository_hash_cache.rb @@ -40,7 +40,11 @@ module Gitlab keys = keys.map { |key| cache_key(key) } Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.unlink(*keys) + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::ClusterUtil.batch_unlink(keys, redis) + else + redis.unlink(*keys) + end end end end diff --git a/lib/gitlab/resource_events/assignment_event_recorder.rb b/lib/gitlab/resource_events/assignment_event_recorder.rb index 94bd05a17ba..0f1ceeb2a66 100644 --- a/lib/gitlab/resource_events/assignment_event_recorder.rb +++ b/lib/gitlab/resource_events/assignment_event_recorder.rb @@ -11,8 +11,6 @@ module Gitlab end def record - return if Feature.disabled?(:record_issue_and_mr_assignee_events, parent.project) - case parent when Issue record_for_parent( diff --git a/lib/gitlab/search/abuse_detection.rb b/lib/gitlab/search/abuse_detection.rb index 7b5377bce88..8711d078ea9 100644 --- a/lib/gitlab/search/abuse_detection.rb +++ b/lib/gitlab/search/abuse_detection.rb @@ -8,7 +8,6 @@ module Gitlab ABUSIVE_TERM_SIZE = 100 ALLOWED_CHARS_REGEX = %r{\A[[:alnum:]_\-\/\.!]+\z}.freeze - MINIMUM_SEARCH_CHARS = 2 ALLOWED_SCOPES = %w( blobs @@ -50,7 +49,8 @@ module Gitlab exclusion: { in: STOP_WORDS, message: 'stopword only abusive search detected' }, allow_blank: true validates :query_string, - length: { minimum: MINIMUM_SEARCH_CHARS, message: 'abusive tiny search detected' }, unless: :skip_tiny_search_validation?, allow_blank: true + length: { minimum: Params::MIN_TERM_LENGTH, message: 'abusive tiny search detected' }, + unless: :skip_tiny_search_validation?, allow_blank: true validates :query_string, no_abusive_term_length: { maximum: ABUSIVE_TERM_SIZE, maximum_for_url: ABUSIVE_TERM_SIZE * 2 } diff --git a/lib/gitlab/search/params.rb b/lib/gitlab/search/params.rb index 1ae14e5e618..6eb24a92be6 100644 --- a/lib/gitlab/search/params.rb +++ b/lib/gitlab/search/params.rb @@ -7,7 +7,7 @@ module Gitlab SEARCH_CHAR_LIMIT = 4096 SEARCH_TERM_LIMIT = 64 - MIN_TERM_LENGTH = 3 + MIN_TERM_LENGTH = 2 # Generic validation validates :query_string, length: { maximum: SEARCH_CHAR_LIMIT } diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 93befc2df57..a733dca6a56 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -107,7 +107,11 @@ module Gitlab def users return User.none unless Ability.allowed?(current_user, :read_users_list) - UsersFinder.new(current_user, search: query).execute + if Feature.enabled?(:autocomplete_users_use_search_service) + UsersFinder.new(current_user, { search: query, use_minimum_char_limit: false }).execute + else + UsersFinder.new(current_user, search: query).execute + end end # highlighting is only performed by Elasticsearch backed results @@ -174,7 +178,9 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def projects - limit_projects.search(query) + scope = limit_projects + scope = scope.non_archived if Feature.enabled?(:search_projects_hide_archived) && !filters[:include_archived] + scope.search(query) end def issues(finder_params = {}) diff --git a/lib/gitlab/sentence.rb b/lib/gitlab/sentence.rb new file mode 100644 index 00000000000..963459e31a3 --- /dev/null +++ b/lib/gitlab/sentence.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Sentence + extend self + # Wraps ActiveSupport's Array#to_sentence to convert the given array to a + # comma-separated sentence joined with localized 'or' Strings instead of 'and'. + def to_exclusive_sentence(array) + array.to_sentence(two_words_connector: _(' or '), last_word_connector: _(', or ')) + end + end +end diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 623b254c4e0..eb73a0a3d31 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -22,10 +22,8 @@ module Gitlab keys_to_expire = keys.map { |key| cache_key(key) } Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - if ::Feature.enabled?(:use_pipeline_over_multikey) - redis.pipelined do |pipeline| - keys_to_expire.each { |key| pipeline.unlink(key) } - end.sum + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::ClusterUtil.batch_unlink(keys_to_expire, redis) else redis.unlink(*keys_to_expire) end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 7ce3f6b5ccb..c4566a6dc2a 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -76,15 +76,19 @@ module Gitlab payload['load_balancing_strategy'] = job['load_balancing_strategy'] if job['load_balancing_strategy'] payload['dedup_wal_locations'] = job['dedup_wal_locations'] if job['dedup_wal_locations'].present? - if job_exception - payload['message'] = "#{message}: fail: #{payload['duration_s']} sec" - payload['job_status'] = 'fail' - - Gitlab::ExceptionLogFormatter.format!(job_exception, payload) - else - payload['message'] = "#{message}: done: #{payload['duration_s']} sec" - payload['job_status'] = 'done' - end + job_status = if job_exception + 'fail' + elsif job['deferred'] + 'deferred' + else + 'done' + end + + payload['message'] = "#{message}: #{job_status}: #{payload['duration_s']} sec" + payload['job_status'] = job_status + payload['job_deferred_by'] = job['deferred_by'] if job['deferred'] + + Gitlab::ExceptionLogFormatter.format!(job_exception, payload) if job_exception db_duration = ActiveRecord::LogSubscriber.runtime payload['db_duration_s'] = Gitlab::Utils.ms_to_round_sec(db_duration) diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index b20f639ce85..ec2a6472809 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -7,7 +7,7 @@ module Gitlab # The result of this method should be passed to # Sidekiq's `config.server_middleware` method # eg: `config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator)` - def self.server_configurator(metrics: true, arguments_logger: true) + def self.server_configurator(metrics: true, arguments_logger: true, defer_jobs: true) lambda do |chain| # Size limiter should be placed at the top chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Server @@ -36,10 +36,11 @@ module Gitlab chain.add ::Gitlab::SidekiqVersioning::Middleware chain.add ::Gitlab::SidekiqStatus::ServerMiddleware chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server - # DuplicateJobs::Server should be placed at the bottom, but before the SidekiqServerMiddleware, + # DuplicateJobs::Server should be placed at the bottom, but before the SidekiqServerMiddleware, # so we can compare the latest WAL location against replica chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server chain.add ::Gitlab::Database::LoadBalancing::SidekiqServerMiddleware + chain.add ::Gitlab::SidekiqMiddleware::DeferJobs if defer_jobs end end diff --git a/lib/gitlab/sidekiq_middleware/defer_jobs.rb b/lib/gitlab/sidekiq_middleware/defer_jobs.rb new file mode 100644 index 00000000000..0a12667865c --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/defer_jobs.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class DeferJobs + DELAY = ENV.fetch("SIDEKIQ_DEFER_JOBS_DELAY", 5.minutes) + FEATURE_FLAG_PREFIX = "defer_sidekiq_jobs" + + DatabaseHealthStatusChecker = Struct.new(:id, :job_class_name) + + # There are 2 scenarios under which this middleware defers a job + # 1. defer_sidekiq_jobs_#{worker_name} FF, jobs are deferred indefinitely until this feature flag + # is turned off or when Feature.enabled? returns false by chance while using `percentage of time` value. + # 2. Gitlab::Database::HealthStatus, on evaluating the db health status if it returns any indicator + # with stop signal, the jobs will be delayed by 'x' seconds (set in worker). + def call(worker, job, _queue) + # ActiveJobs have wrapped class stored in 'wrapped' key + resolved_class = job['wrapped']&.safe_constantize || worker.class + defer_job, delay, deferred_by = defer_job_info(resolved_class, job) + + if !!defer_job + # Referred in job_logger's 'log_job_done' method to compute proper 'job_status' + job['deferred'] = true + job['deferred_by'] = deferred_by + + worker.class.perform_in(delay, *job['args']) + counter.increment({ worker: worker.class.name }) + + # This breaks the middleware chain and return + return + end + + yield + end + + private + + def defer_job_info(worker_class, job) + if defer_job_by_ff?(worker_class) + [true, DELAY, :feature_flag] + elsif defer_job_by_database_health_signal?(job, worker_class) + [true, worker_class.database_health_check_attrs[:delay_by], :database_health_check] + end + end + + def defer_job_by_ff?(worker_class) + Feature.enabled?( + :"#{FEATURE_FLAG_PREFIX}_#{worker_class.name}", + type: :worker, + default_enabled_if_undefined: false + ) + end + + def defer_job_by_database_health_signal?(job, worker_class) + unless worker_class.respond_to?(:defer_on_database_health_signal?) && + worker_class.defer_on_database_health_signal? + return false + end + + health_check_attrs = worker_class.database_health_check_attrs + job_base_model = Gitlab::Database.schemas_to_base_models[health_check_attrs[:gitlab_schema]].first + + health_context = Gitlab::Database::HealthStatus::Context.new( + DatabaseHealthStatusChecker.new(job['jid'], worker_class.name), + job_base_model.connection, + health_check_attrs[:gitlab_schema], + health_check_attrs[:tables] + ) + + Gitlab::Database::HealthStatus.evaluate(health_context).any?(&:stop?) + end + + def counter + @counter ||= Gitlab::Metrics.counter(:sidekiq_jobs_deferred_total, 'The number of jobs deferred') + end + end + end +end diff --git a/lib/gitlab/silent_mode.rb b/lib/gitlab/silent_mode.rb new file mode 100644 index 00000000000..7c7cbf8f1d9 --- /dev/null +++ b/lib/gitlab/silent_mode.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module SilentMode + def self.enabled? + Gitlab::CurrentSettings.silent_mode_enabled? + end + + def self.log_info(data) + Gitlab::AppJsonLogger.info(**add_silent_mode_log_data(data)) + end + + def self.log_debug(data) + Gitlab::AppJsonLogger.debug(**add_silent_mode_log_data(data)) + end + + def self.add_silent_mode_log_data(data) + data.merge!({ silent_mode_enabled: enabled? }) + end + end +end diff --git a/lib/gitlab/slash_commands/incident_management/incident_command.rb b/lib/gitlab/slash_commands/incident_management/incident_command.rb index 3fa08621777..1c870efe8b1 100644 --- a/lib/gitlab/slash_commands/incident_management/incident_command.rb +++ b/lib/gitlab/slash_commands/incident_management/incident_command.rb @@ -11,9 +11,13 @@ module Gitlab def collection IssuesFinder.new(current_user, project_id: project.id, issue_types: :incident).execute end + + def slack_installation + slack_workspace_id = params[:team_id] + + SlackIntegration.with_bot.find_by_team_id(slack_workspace_id) + end end end end end - -Gitlab::SlashCommands::IncidentManagement::IncidentCommand.prepend_mod diff --git a/lib/gitlab/slash_commands/incident_management/incident_new.rb b/lib/gitlab/slash_commands/incident_management/incident_new.rb index a43235bdeb6..b5c43873355 100644 --- a/lib/gitlab/slash_commands/incident_management/incident_new.rb +++ b/lib/gitlab/slash_commands/incident_management/incident_new.rb @@ -16,6 +16,14 @@ module Gitlab text == 'incident declare' end + def execute(_match) + response = ::Integrations::SlackInteractions::IncidentManagement::IncidentModalOpenedService + .new(slack_installation, current_user, params) + .execute + + presenter.present(response.message) + end + private def presenter diff --git a/lib/gitlab/slash_commands/issue_new.rb b/lib/gitlab/slash_commands/issue_new.rb index 508526ac500..dd2bf632e2c 100644 --- a/lib/gitlab/slash_commands/issue_new.rb +++ b/lib/gitlab/slash_commands/issue_new.rb @@ -37,7 +37,7 @@ module Gitlab private def create_issue(title:, description:) - ::Issues::CreateService.new(container: project, current_user: current_user, params: { title: title, description: description }, spam_params: nil).execute + ::Issues::CreateService.new(container: project, current_user: current_user, params: { title: title, description: description }, perform_spam_check: false).execute end def presenter(issue) diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index 222dd54b7b4..0afaf46fa9b 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -58,6 +58,7 @@ module Gitlab pb.title = spammable.spam_title || '' if pb.respond_to?(:title) pb.description = spammable.spam_description || '' if pb.respond_to?(:description) pb.text = spammable.spammable_text || '' if pb.respond_to?(:text) + pb.type = spammable.spammable_entity_type if pb.respond_to?(:type) pb.created_at = convert_to_pb_timestamp(spammable.created_at) if spammable.created_at pb.updated_at = convert_to_pb_timestamp(spammable.updated_at) if spammable.updated_at pb.action = ACTION_MAPPING.fetch(context.fetch(:action)) if context.has_key?(:action) diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb index 6d2677175e6..a6ab36e6cd2 100644 --- a/lib/gitlab/template/finders/global_template_finder.rb +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -25,7 +25,7 @@ module Gitlab # The key is untrusted input, so ensure we can't be directed outside # of base_dir - Gitlab::Utils.check_path_traversal!(file_name) + Gitlab::PathTraversal.check_path_traversal!(file_name) directory = select_directory(file_name) directory ? File.join(category_directory(directory), file_name) : nil diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index 9f0ba97bcdf..8343750e04a 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -29,7 +29,7 @@ module Gitlab # The key is untrusted input, so ensure we can't be directed outside # of base_dir inside the repository - Gitlab::Utils.check_path_traversal!(file_name) + Gitlab::PathTraversal.check_path_traversal!(file_name) directory = select_directory(file_name) raise FileNotFoundError if directory.nil? diff --git a/lib/gitlab/template/metrics_dashboard_template.rb b/lib/gitlab/template/metrics_dashboard_template.rb deleted file mode 100644 index 469f97d7cb1..00000000000 --- a/lib/gitlab/template/metrics_dashboard_template.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Template - class MetricsDashboardTemplate < BaseTemplate - def description - "# This file is a template, and might need editing before it works on your project." - end - - class << self - def extension - '.metrics-dashboard.yml' - end - - def categories - { - "General" => '' - } - end - - def base_dir - Rails.root.join('lib/gitlab/metrics/templates') - end - - def finder(project = nil) - Gitlab::Template::Finders::GlobalTemplateFinder.new(self.base_dir, self.extension, self.categories) - end - end - end - end -end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 52aee4d2d45..f127e14243c 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -68,6 +68,14 @@ module Gitlab false end + def tracker + @tracker ||= if snowplow_micro_enabled? + Gitlab::Tracking::Destinations::SnowplowMicro.new + else + Gitlab::Tracking::Destinations::Snowplow.new + end + end + private def track_struct_event(destination, category, action, label:, property:, value:, contexts:) # rubocop:disable Metrics/ParameterLists @@ -76,14 +84,6 @@ module Gitlab rescue StandardError => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) end - - def tracker - @tracker ||= if snowplow_micro_enabled? - Gitlab::Tracking::Destinations::SnowplowMicro.new - else - Gitlab::Tracking::Destinations::Snowplow.new - end - end end end end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 62c45368410..61d6fdc6dca 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,7 +3,7 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-8' + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-9' GITLAB_RAILS_SOURCE = 'gitlab-rails' def initialize(namespace_id: nil, plan_name: nil, project_id: nil, user_id: nil, **extra) diff --git a/lib/gitlab/usage/metrics/instrumentations/count_all_ci_builds_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_all_ci_builds_metric.rb new file mode 100644 index 00000000000..2a3cbaf5d03 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_all_ci_builds_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountAllCiBuildsMetric < DatabaseMetric + operation :count + + relation { ::Ci::Build } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_deployments_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_deployments_metric.rb new file mode 100644 index 00000000000..97813fbb5c0 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_deployments_metric.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountDeploymentsMetric < DatabaseMetric + operation :count + + start { Deployment.minimum(:id) } + finish { Deployment.maximum(:id) } + + def initialize(metric_definition) + super + + raise ArgumentError, 'Missing Deployment type' unless type + raise ArgumentError, "Invalid Deployment type: #{type}" unless type.in?(%i[all success failed]) + end + + private + + def type + options[:type].to_sym + end + + def relation + @metric_relation = case type + when :all + Deployment + when :success + Deployment.success + when :failed + Deployment.failed + end.where(time_constraints) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb index d485e8b4f72..05e29f2d885 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_imported_projects_metric.rb @@ -23,7 +23,7 @@ module Gitlab ::Project .select(:id) .where(Project.arel_table[:created_at].gteq(start)) # rubocop:disable UsageData/LargeTable - .order(created_at: :asc).limit(1).first&.id + .order(created_at: :asc).order(id: :asc).limit(1).first&.id end end end @@ -36,7 +36,7 @@ module Gitlab ::Project .select(:id) .where(Project.arel_table[:created_at].lteq(finish)) # rubocop:disable UsageData/LargeTable - .order(created_at: :desc).limit(1).first&.id + .order(created_at: :desc).order(id: :desc).limit(1).first&.id end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_personal_snippets_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_personal_snippets_metric.rb new file mode 100644 index 00000000000..9a34c535676 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_personal_snippets_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountPersonalSnippetsMetric < DatabaseMetric + operation :count + + relation do + PersonalSnippet + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_project_snippets_metric.rb index c2ca62f9eba..af25a32592c 100644 --- a/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_project_snippets_metric.rb @@ -4,9 +4,11 @@ module Gitlab module Usage module Metrics module Instrumentations - class InstallationCreationDateMetric < GenericMetric - value do - User.where(id: 1).pick(:created_at) + class CountProjectSnippetsMetric < DatabaseMetric + operation :count + + relation do + ProjectSnippet end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_projects_with_alerts_created_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_alerts_created_metric.rb new file mode 100644 index 00000000000..8ae4000b802 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_alerts_created_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountProjectsWithAlertsCreatedMetric < DatabaseMetric + operation :distinct_count, column: :project_id + + relation do + ::AlertManagement::Alert + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_snippets_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_snippets_metric.rb new file mode 100644 index 00000000000..342ba802fd8 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_snippets_metric.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountSnippetsMetric < DatabaseMetric + operation :count + # Relation and operation are not used, but are included to satisfy expectations + # of other metric generation logic. + relation { Snippet } + + def value + count(project_snippet_relation) + count(personal_snippet_relation) + end + + def project_snippet_relation + ProjectSnippet.where(time_constraints) + end + + def personal_snippet_relation + PersonalSnippet.where(time_constraints) + end + + def to_sql + project_snippet_relation_sql = Gitlab::Usage::Metrics::Query.for(:count, project_snippet_relation) + personal_snippet_relation_sql = Gitlab::Usage::Metrics::Query.for(:count, personal_snippet_relation) + + "SELECT (#{project_snippet_relation_sql}) + (#{personal_snippet_relation_sql})" + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 846bb934a3d..72168bce782 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -42,14 +42,11 @@ module Gitlab clear_memoized with_finished_at(:recording_ce_finished_at) do - usage_data_metrics + { recorded_at: recorded_at } + .merge(usage_data_metrics) end end - def license_usage_data - { recorded_at: recorded_at } - end - def recorded_at @recorded_at ||= Time.current end @@ -70,9 +67,6 @@ module Gitlab auto_devops_disabled: count(::ProjectAutoDevops.disabled), deploy_keys: count(DeployKey), # rubocop: disable UsageData/LargeTable: - deployments: deployment_count(Deployment), - successful_deployments: deployment_count(Deployment.success), - failed_deployments: deployment_count(Deployment.failed), feature_flags: count(Operations::FeatureFlag), # rubocop: enable UsageData/LargeTable: environments: count(::Environment), @@ -101,7 +95,7 @@ module Gitlab issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data, issues_created_from_alerts: total_alert_issues, - incident_issues: count(::Issue.incident, start: minimum_id(Issue), finish: maximum_id(Issue)), + incident_issues: count(::Issue.with_issue_type(:incident), start: minimum_id(Issue), finish: maximum_id(Issue)), alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)), keys: count(Key), label_lists: count(List.label), @@ -113,11 +107,10 @@ module Gitlab pages_domains: count(PagesDomain), pool_repositories: count(PoolRepository), projects: count(Project), - projects_creating_incidents: distinct_count(Issue.incident, :project_id), + projects_creating_incidents: distinct_count(Issue.with_issue_type(:incident), :project_id), projects_imported_from_github: count(Project.where(import_type: 'github')), projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), - projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.of_report_type(:terraform), :project_id), projects_with_terraform_states: distinct_count(::Terraform::State, :project_id), @@ -125,8 +118,6 @@ module Gitlab protected_branches_except_default: count(ProtectedBranch.where.not(name: ['main', 'master', Gitlab::CurrentSettings.default_branch_name])), releases: count(Release), remote_mirrors: count(RemoteMirror), - personal_snippets: count(PersonalSnippet), - project_snippets: count(ProjectSnippet), suggestions: count(Suggestion), terraform_reports: count(::Ci::JobArtifact.of_report_type(:terraform)), terraform_states: count(::Terraform::State), @@ -140,9 +131,7 @@ module Gitlab integrations_usage, user_preferences_usage, service_desk_counts - ).tap do |data| - data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) - end + ) } end # rubocop: enable Metrics/AbcSize @@ -150,19 +139,9 @@ module Gitlab def system_usage_data_monthly { counts_monthly: { - # rubocop: disable UsageData/LargeTable: - deployments: deployment_count(Deployment.where(monthly_time_range_db_params)), - successful_deployments: deployment_count(Deployment.success.where(monthly_time_range_db_params)), - failed_deployments: deployment_count(Deployment.failed.where(monthly_time_range_db_params)), - # rubocop: enable UsageData/LargeTable: projects: count(Project.where(monthly_time_range_db_params), start: minimum_id(Project), finish: maximum_id(Project)), - packages: count(::Packages::Package.where(monthly_time_range_db_params)), - personal_snippets: count(PersonalSnippet.where(monthly_time_range_db_params)), - project_snippets: count(ProjectSnippet.where(monthly_time_range_db_params)), - projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(monthly_time_range_db_params), :project_id) - }.tap do |data| - data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) - end + packages: count(::Packages::Package.where(monthly_time_range_db_params)) + } } end # rubocop: enable CodeReuse/ActiveRecord @@ -372,7 +351,6 @@ module Gitlab package: usage_activity_by_stage_package(time_period), plan: usage_activity_by_stage_plan(time_period), release: usage_activity_by_stage_release(time_period), - secure: usage_activity_by_stage_secure(time_period), verify: usage_activity_by_stage_verify(time_period) } } @@ -450,8 +428,11 @@ module Gitlab start: minimum_id(User), finish: maximum_id(User)), projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id), - projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id), - projects_with_alert_incidents: distinct_count(::Issue.incident.with_alert_management_alerts.where(time_period), :project_id), + projects_with_incidents: distinct_count(::Issue.with_issue_type(:incident).where(time_period), :project_id), + # We are making an assumption here that all alert_management_alerts are associated with an issue of type + # incident. In reality this is very close to the truth and allows more efficient queries. + # More info in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/121297#note_1416999956 + projects_with_alert_incidents: distinct_count(::AlertManagement::Alert.where(time_period).where.not(issue_id: nil), :project_id), projects_with_enabled_alert_integrations_histogram: integrations_histogram }.compact end @@ -514,13 +495,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # Currently too complicated and to get reliable counts for these stats: - # container_scanning_jobs, dast_jobs, dependency_scanning_jobs, license_management_jobs, sast_jobs, secret_detection_jobs - # Once https://gitlab.com/gitlab-org/gitlab/merge_requests/17568 is merged, this might be doable - def usage_activity_by_stage_secure(time_period) - {} - end - def with_metadata result = nil error = nil @@ -552,8 +526,7 @@ module Gitlab end def usage_data_metrics - license_usage_data - .merge(system_usage_data_license) + system_usage_data_license .merge(system_usage_data_settings) .merge(system_usage_data) .merge(system_usage_data_monthly) @@ -637,10 +610,6 @@ module Gitlab omniauth_provider_names.reject { |name| name.starts_with?('ldap') } end - def deployment_count(relation) - count relation, start: minimum_id(Deployment), finish: maximum_id(Deployment) - end - def project_imports(time_period) time_frame = metric_time_period(time_period) counters = { diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index badcda1def0..eaa4bf15fe1 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -3,18 +3,14 @@ module Gitlab module UsageDataCounters module HLLRedisCounter - DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks - DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days + KEY_EXPIRY_LENGTH = 6.weeks REDIS_SLOT = 'hll_counters' EventError = Class.new(StandardError) UnknownEvent = Class.new(EventError) - UnknownAggregation = Class.new(EventError) - AggregationMismatch = Class.new(EventError) InvalidContext = Class.new(EventError) KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) - ALLOWED_AGGREGATIONS = %i(daily weekly).freeze # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id @@ -24,7 +20,6 @@ module Gitlab # Event example: # # - name: g_compliance_dashboard # Unique event name - # aggregation: weekly # Aggregation level, keys are stored weekly # # Usage: # @@ -63,8 +58,7 @@ module Gitlab # end_date - The end date of the time range. # context - Event context, plan level tracking. Available if set when tracking. def unique_events(event_names:, start_date:, end_date:, context: '') - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events| - raise AggregationMismatch, events unless events_same_aggregation?(events) + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do raise InvalidContext if context.present? && !context.in?(valid_context_list) end end @@ -78,9 +72,7 @@ module Gitlab end def calculate_events_union(event_names:, start_date:, end_date:) - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| - raise AggregationMismatch, events unless events_same_aggregation?(events) - end + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) end private @@ -94,12 +86,7 @@ module Gitlab return if event.blank? return unless Feature.enabled?(:redis_hll_tracking, type: :ops) - if event[:aggregation].to_sym == :daily - weekly_event = event.dup.tap { |e| e['aggregation'] = 'weekly' } - Gitlab::Redis::HLL.add(key: redis_key(weekly_event, time, context), value: values, expiry: expiry(weekly_event)) - end - - Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) + Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: KEY_EXPIRY_LENGTH) rescue StandardError => e # Ignore any exceptions unless is dev or test env @@ -117,25 +104,18 @@ module Gitlab yield events if block_given? - aggregation = events.first[:aggregation] - - if Feature.disabled?(:revert_daily_hll_events_to_weekly_aggregation) - aggregation = 'weekly' - events = events.map { |e| e.merge(aggregation: 'weekly') } - end + keys = keys_for_aggregation(events: events, start_date: start_date, end_date: end_date, context: context) - keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context) return FALLBACK unless keys.any? redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end - def keys_for_aggregation(aggregation, events:, start_date:, end_date:, context: '') - if aggregation.to_sym == :daily - daily_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) - else - weekly_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) - end + def keys_for_aggregation(events:, start_date:, end_date:, context: '') + end_date = end_date.end_of_week - 1.week + (start_date.to_date..end_date.to_date).map do |date| + events.map { |event| redis_key(event, date, context) } + end.flatten.uniq end def load_events(wildcard) @@ -152,15 +132,6 @@ module Gitlab known_events.map { |event| event[:name] } end - def events_same_aggregation?(events) - aggregation = events.first[:aggregation] - events.all? { |event| event[:aggregation] == aggregation } - end - - def expiry(event) - event[:aggregation].to_sym == :daily ? DEFAULT_DAILY_KEY_EXPIRY_LENGTH : DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH - end - def event_for(event_name) known_events.find { |event| event[:name] == event_name.to_s } end @@ -173,36 +144,13 @@ module Gitlab def redis_key(event, time, context = '') raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s) - # ToDo: remove during https://gitlab.com/groups/gitlab-org/-/epics/9542 cleanup - raise UnknownAggregation, "Use :daily or :weekly aggregation" unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) - key = "{#{REDIS_SLOT}}_#{event[:name]}" - key = apply_time_aggregation(key, time, event) - key = "#{context}_#{key}" if context.present? - key - end - def apply_time_aggregation(key, time, event) - if event[:aggregation].to_sym == :daily - year_day = time.strftime('%G-%j') - "#{year_day}-#{key}" - else - year_week = time.strftime('%G-%V') - "#{key}-#{year_week}" - end - end + year_week = time.strftime('%G-%V') + key = "#{key}-#{year_week}" - def daily_redis_keys(events:, start_date:, end_date:, context: '') - (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date, context) } - end.flatten - end - - def weekly_redis_keys(events:, start_date:, end_date:, context: '') - end_date = end_date.end_of_week - 1.week - (start_date.to_date..end_date.to_date).map do |date| - events.map { |event| redis_key(event, date, context) } - end.flatten.uniq + key = "#{context}_#{key}" if context.present? + key end end end diff --git a/lib/gitlab/usage_data_counters/jetbrains_bundled_plugin_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/jetbrains_bundled_plugin_activity_unique_counter.rb new file mode 100644 index 00000000000..a9e8d9bf0cb --- /dev/null +++ b/lib/gitlab/usage_data_counters/jetbrains_bundled_plugin_activity_unique_counter.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module JetBrainsBundledPluginActivityUniqueCounter + JETBRAINS_BUNDLED_API_REQUEST_ACTION = 'i_editor_extensions_user_jetbrains_bundled_api_request' + JETBRAINS_BUNDLED_USER_AGENT_REGEX = /\AIntelliJ-GitLab-Plugin/ + + class << self + def track_api_request_when_trackable(user_agent:, user:) + user_agent&.match?(JETBRAINS_BUNDLED_USER_AGENT_REGEX) && + track_unique_action_by_user(JETBRAINS_BUNDLED_API_REQUEST_ACTION, user) + end + + private + + def track_unique_action_by_user(action, user) + return unless user + + track_unique_action(action, user.id) + end + + def track_unique_action(action, value) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index f685f0d65d9..c3e1c34151b 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -307,3 +307,5 @@ aggregation: weekly - name: p_ci_templates_terraform_module aggregation: weekly +- name: p_ci_templates_pages_zola + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index db0c0653f63..bd8c79f4801 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -79,6 +79,8 @@ aggregation: weekly - name: i_code_review_user_jetbrains_api_request aggregation: weekly +- name: i_editor_extensions_user_jetbrains_bundled_api_request + aggregation: weekly - name: i_code_review_user_gitlab_cli_api_request aggregation: weekly - name: i_code_review_user_create_mr_from_issue diff --git a/lib/gitlab/usage_data_counters/known_events/product_analytics.yml b/lib/gitlab/usage_data_counters/known_events/product_analytics.yml index 5a791c4b3c2..c43bf9040dd 100644 --- a/lib/gitlab/usage_data_counters/known_events/product_analytics.yml +++ b/lib/gitlab/usage_data_counters/known_events/product_analytics.yml @@ -2,3 +2,7 @@ aggregation: weekly - name: project_initialized_product_analytics aggregation: weekly +- name: user_created_analytics_dashboard + aggregation: weekly +- name: user_visited_dashboard + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 136d284f462..69f92ac5c0a 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -133,3 +133,7 @@ aggregation: weekly - name: i_quickactions_blocks aggregation: weekly +- name: i_quickactions_unlink + aggregation: weekly +- name: i_quickactions_promote_to + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/workspaces.yml b/lib/gitlab/usage_data_counters/known_events/workspaces.yml new file mode 100644 index 00000000000..8a96524b167 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/workspaces.yml @@ -0,0 +1,5 @@ +- name: users_updating_workspaces + aggregation: weekly + +- name: users_creating_workspaces + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb index d6e05f30a0d..ece2ffea83b 100644 --- a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb +++ b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb @@ -4,7 +4,7 @@ module Gitlab module UsageDataCounters class KubernetesAgentCounter < BaseCounter PREFIX = 'kubernetes_agent' - KNOWN_EVENTS = %w[gitops_sync k8s_api_proxy_request].freeze + KNOWN_EVENTS = %w[gitops_sync k8s_api_proxy_request flux_git_push_notifications_total].freeze class << self def increment_event_counts(events) diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index fceeacb60ca..1ed2e891a1f 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -64,20 +64,15 @@ module Gitlab end def track_create_mr_action(user:, merge_request:) - track_unique_action_by_user(MR_USER_CREATE_ACTION, user) track_unique_action_by_merge_request(MR_CREATE_ACTION, merge_request) project = merge_request.target_project - Gitlab::Tracking.event( - name, - :create, - project: project, - namespace: project.namespace, - user: user, - property: MR_USER_CREATE_ACTION, - label: 'redis_hll_counters.code_review.i_code_review_user_create_mr_monthly', - context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, - event: MR_USER_CREATE_ACTION).to_context] + + Gitlab::InternalEvents.track_event( + MR_USER_CREATE_ACTION, + user_id: user.id, + project_id: project.id, + namespace_id: project.namespace_id ) end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index b92e7dbb725..dc0112c14d6 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -3,34 +3,8 @@ module Gitlab module Utils extend self - PathTraversalAttackError ||= Class.new(StandardError) DoubleEncodingError ||= Class.new(StandardError) - private_class_method def logger - @logger ||= Gitlab::AppLogger - end - - # Ensure that the relative path will not traverse outside the base directory - # We url decode the path to avoid passing invalid paths forward in url encoded format. - # Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580 - # It also checks for ALT_SEPARATOR aka '\' (forward slash) - def check_path_traversal!(path) - return unless path - - path = path.to_s if path.is_a?(Gitlab::HashedPath) - raise PathTraversalAttackError, 'Invalid path' unless path.is_a?(String) - - path = decode_path(path) - path_regex = %r{(\A(\.{1,2})\z|\A\.\.[/\\]|[/\\]\.\.\z|[/\\]\.\.[/\\]|\n)} - - if path.match?(path_regex) - logger.warn(message: "Potential path traversal attempt detected", path: "#{path}") - raise PathTraversalAttackError, 'Invalid path' - end - - path - end - def allowlisted?(absolute_path, allowlist) path = absolute_path.downcase @@ -39,20 +13,6 @@ module Gitlab end end - def check_allowed_absolute_path!(path, allowlist) - return unless Pathname.new(path).absolute? - return if allowlisted?(path, allowlist) - - raise StandardError, "path #{path} is not allowed" - end - - def check_allowed_absolute_path_and_path_traversal!(path, path_allowlist) - traversal_path = check_path_traversal!(path) - raise StandardError, "path is not a string!" unless traversal_path.is_a?(String) - - check_allowed_absolute_path!(traversal_path, path_allowlist) - end - def decode_path(encoded_path) decoded = CGI.unescape(encoded_path) if decoded != CGI.unescape(decoded) @@ -103,12 +63,6 @@ module Gitlab .gsub(/(\A-+|-+\z)/, '') end - # Wraps ActiveSupport's Array#to_sentence to convert the given array to a - # comma-separated sentence joined with localized 'or' Strings instead of 'and'. - def to_exclusive_sentence(array) - array.to_sentence(two_words_connector: _(' or '), last_word_connector: _(', or ')) - end - # Converts newlines into HTML line break elements def nlbr(str) ActionView::Base.full_sanitizer.sanitize(+str, tags: []).gsub(/\r?\n/, '<br>').html_safe diff --git a/lib/gitlab/utils/markdown.rb b/lib/gitlab/utils/markdown.rb index 5087020affe..c95398a15df 100644 --- a/lib/gitlab/utils/markdown.rb +++ b/lib/gitlab/utils/markdown.rb @@ -4,7 +4,7 @@ module Gitlab module Utils module Markdown PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze - PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate|free|bronze|silver|gold)(\s+(only|self|sass))?\)\**/.freeze + PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate|free|bronze|silver|gold)(\s+(only|self|saas))?\)\**/.freeze def string_to_anchor(string) string diff --git a/lib/gitlab/utils/sanitize_node_link.rb b/lib/gitlab/utils/sanitize_node_link.rb index 9c34302f75e..183741ff5f5 100644 --- a/lib/gitlab/utils/sanitize_node_link.rb +++ b/lib/gitlab/utils/sanitize_node_link.rb @@ -51,7 +51,9 @@ module Gitlab begin node[attr] = node[attr].strip + uri = Addressable::URI.parse(node[attr]) + uri = uri.normalize next unless uri.scheme next if safe_protocol?(uri.scheme) diff --git a/lib/gitlab/verify/ci_secure_files.rb b/lib/gitlab/verify/ci_secure_files.rb new file mode 100644 index 00000000000..9bb7f7260c4 --- /dev/null +++ b/lib/gitlab/verify/ci_secure_files.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Verify + class CiSecureFiles < BatchVerifier + def name + 'CI Secure Files' + end + + def describe(object) + "SecureFile: #{object.id}" + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def all_relation + ::Ci::SecureFile.all + end + # rubocop: enable CodeReuse/ActiveRecord + + def local?(secure_file) + secure_file.local? + end + + def expected_checksum(secure_file) + secure_file.checksum + end + + def actual_checksum(secure_file) + Digest::SHA256.hexdigest(secure_file.file.read) + end + + def remote_object_exists?(secure_file) + secure_file.file.file.exists? + end + end + end +end diff --git a/lib/gitlab/x509/tag.rb b/lib/gitlab/x509/tag.rb index cf24e6f62bd..b9a3b11ac34 100644 --- a/lib/gitlab/x509/tag.rb +++ b/lib/gitlab/x509/tag.rb @@ -11,7 +11,7 @@ module Gitlab strong_memoize(:signature) do super - signature = X509::Signature.new(signature_text, signed_text, @tag.tagger.email, Time.at(@tag.tagger.date.seconds)) + signature = X509::Signature.new(signature_text, signed_text, @tag.user_email, @tag.date) signature unless signature.verified_signature.nil? end end |