diff options
Diffstat (limited to 'lib/gitlab')
208 files changed, 3660 insertions, 1219 deletions
diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 722ee061eba..0c52ce8aba4 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -37,6 +37,7 @@ module Gitlab users_get_by_id: { threshold: -> { application_settings.users_get_by_id_limit }, interval: 10.minutes }, username_exists: { threshold: 20, interval: 1.minute }, user_sign_up: { threshold: 20, interval: 1.minute }, + user_sign_in: { threshold: 5, interval: 10.minutes }, profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, profile_update_username: { threshold: 10, interval: 1.minute }, update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, @@ -45,7 +46,10 @@ module Gitlab search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, gitlab_shell_operation: { threshold: 600, interval: 1.minute }, pipelines_create: { threshold: -> { application_settings.pipeline_limit_per_project_user_sha }, interval: 1.minute }, - temporary_email_failure: { threshold: 50, interval: 1.day } + temporary_email_failure: { threshold: 50, interval: 1.day }, + project_testing_integration: { threshold: 5, interval: 1.minute }, + email_verification: { threshold: 10, interval: 10.minutes }, + email_verification_code_send: { threshold: 10, interval: 1.hour } }.freeze end @@ -53,15 +57,26 @@ module Gitlab # be throttled. # # @param key [Symbol] Key attribute registered in `.rate_limits` - # @param scope [Array<ActiveRecord>] Array of ActiveRecord models, Strings or Symbols to scope throttling to a specific request (e.g. per user per project) - # @param threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` - # @param users_allowlist [Array<String>] Optional list of usernames to exclude from the limit. This param will only be functional if Scope includes a current user. - # @param peek [Boolean] Optional. When true the key will not be incremented but the current throttled state will be returned. + # @param scope [Array<ActiveRecord>] Array of ActiveRecord models, Strings + # or Symbols to scope throttling to a specific request (e.g. per user + # per project) + # @param resource [ActiveRecord] An ActiveRecord model to count an action + # for (e.g. limit unique project (resource) downloads (action) to five + # per user (scope)) + # @param threshold [Integer] Optional threshold value to override default + # one registered in `.rate_limits` + # @param users_allowlist [Array<String>] Optional list of usernames to + # exclude from the limit. This param will only be functional if Scope + # includes a current user. + # @param peek [Boolean] Optional. When true the key will not be + # incremented but the current throttled state will be returned. # # @return [Boolean] Whether or not a request should be throttled - def throttled?(key, scope:, threshold: nil, users_allowlist: nil, peek: false) + def throttled?(key, scope:, resource: nil, threshold: nil, users_allowlist: nil, peek: false) raise InvalidKeyError unless rate_limits[key] + strategy = resource.present? ? IncrementPerActionedResource.new(resource.id) : IncrementPerAction.new + ::Gitlab::Instrumentation::RateLimitingGates.track(key) return false if scoped_user_in_allowlist?(scope, users_allowlist) @@ -71,6 +86,9 @@ module Gitlab return false if threshold_value == 0 interval_value = interval(key) + + return false if interval_value == 0 + # `period_key` is based on the current time and interval so when time passes to the next interval # the key changes and the rate limit count starts again from 0. # Based on https://github.com/rack/rack-attack/blob/886ba3a18d13c6484cd511a4dc9b76c0d14e5e96/lib/rack/attack/cache.rb#L63-L68 @@ -78,9 +96,12 @@ module Gitlab cache_key = cache_key(key, scope, period_key) value = if peek - read(cache_key) + strategy.read(cache_key) else - increment(cache_key, interval_value, time_elapsed_in_period) + # We add a 1 second buffer to avoid timing issues when we're at the end of a period + expiry = interval_value - time_elapsed_in_period + 1 + + strategy.increment(cache_key, expiry) end value > threshold_value @@ -128,40 +149,25 @@ module Gitlab def threshold(key) value = rate_limit_value_by_key(key, :threshold) - return value.call if value.is_a?(Proc) - - value.to_i + rate_limit_value(value) end def interval(key) - rate_limit_value_by_key(key, :interval).to_i - end - - def rate_limit_value_by_key(key, setting) - action = rate_limits[key] + value = rate_limit_value_by_key(key, :interval) - action[setting] if action + rate_limit_value(value) end - # Increments the rate limit count and returns the new count value. - def increment(cache_key, interval_value, time_elapsed_in_period) - # We add a 1 second buffer to avoid timing issues when we're at the end of a period - expiry = interval_value - time_elapsed_in_period + 1 + def rate_limit_value(value) + value = value.call if value.is_a?(Proc) - ::Gitlab::Redis::RateLimiting.with do |redis| - redis.pipelined do - redis.incr(cache_key) - redis.expire(cache_key, expiry) - end.first - end + value.to_i end - # Returns the rate limit count. - # Will be 0 if there is no data in the cache. - def read(cache_key) - ::Gitlab::Redis::RateLimiting.with do |redis| - redis.get(cache_key).to_i - end + def rate_limit_value_by_key(key, setting) + action = rate_limits[key] + + action[setting] if action end def cache_key(key, scope, period_key) diff --git a/lib/gitlab/application_rate_limiter/base_strategy.rb b/lib/gitlab/application_rate_limiter/base_strategy.rb new file mode 100644 index 00000000000..b97770c0524 --- /dev/null +++ b/lib/gitlab/application_rate_limiter/base_strategy.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module ApplicationRateLimiter + class BaseStrategy + # Increment the rate limit count and return the new count value + def increment(cache_key, expiry) + raise NotImplementedError + end + + # Return the rate limit count. + # Should be 0 if there is no data in the cache. + def read(cache_key) + raise NotImplementedError + end + + private + + def with_redis(&block) + ::Gitlab::Redis::RateLimiting.with(&block) # rubocop: disable CodeReuse/ActiveRecord + end + end + end +end diff --git a/lib/gitlab/application_rate_limiter/increment_per_action.rb b/lib/gitlab/application_rate_limiter/increment_per_action.rb new file mode 100644 index 00000000000..c99d03f1344 --- /dev/null +++ b/lib/gitlab/application_rate_limiter/increment_per_action.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module ApplicationRateLimiter + class IncrementPerAction < BaseStrategy + def increment(cache_key, expiry) + with_redis do |redis| + redis.pipelined do + redis.incr(cache_key) + redis.expire(cache_key, expiry) + end.first + end + end + + def read(cache_key) + with_redis do |redis| + redis.get(cache_key).to_i + end + end + end + end +end diff --git a/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb new file mode 100644 index 00000000000..8b4197cfff9 --- /dev/null +++ b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module ApplicationRateLimiter + class IncrementPerActionedResource < BaseStrategy + def initialize(resource_key) + @resource_key = resource_key + end + + def increment(cache_key, expiry) + with_redis do |redis| + redis.pipelined do + redis.sadd(cache_key, resource_key) + redis.expire(cache_key, expiry) + redis.scard(cache_key) + end.last + end + end + + def read(cache_key) + with_redis do |redis| + redis.scard(cache_key) + end + end + + private + + attr_accessor :resource_key + end + end +end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 5d5a431f206..6c3487c28ea 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -217,7 +217,7 @@ module Gitlab return unless valid_scoped_token?(token, all_available_scopes) if project && token.user.project_bot? - return unless token_bot_in_resource?(token.user, project) + return unless can_read_project?(token.user, project) end if token.user.can_log_in_with_non_expired_password? || token.user.project_bot? @@ -225,22 +225,8 @@ module Gitlab end end - def token_bot_in_project?(user, project) - project.bots.include?(user) - end - - # rubocop: disable CodeReuse/ActiveRecord - - # A workaround for adding group-level automation is to add the bot user of a project access token as a group member. - # In order to make project access tokens work this way during git authentication, we need to add an additional check for group membership. - # This is a temporary workaround until service accounts are implemented. - def token_bot_in_group?(user, project) - project.group && project.group.members_with_parents.where(user_id: user.id).exists? - end - # rubocop: enable CodeReuse/ActiveRecord - - def token_bot_in_resource?(user, project) - token_bot_in_project?(user, project) || token_bot_in_group?(user, project) + def can_read_project?(user, project) + user.can?(:read_project, project) end def valid_oauth_token?(token) @@ -323,7 +309,7 @@ module Gitlab return unless build.project.builds_enabled? if build.user - return unless build.user.can_log_in_with_non_expired_password? || (build.user.project_bot? && token_bot_in_resource?(build.user, build.project)) + return unless build.user.can_log_in_with_non_expired_password? || (build.user.project_bot? && can_read_project?(build.user, build.project)) # If user is assigned to build, use restricted credentials of user Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities) diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index 56c2af1910e..957ec5fa479 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -31,7 +31,11 @@ module Gitlab end def valid_sign_in? - allowed? && super + # The order is important here: we need to ensure the + # associated GitLab user entry is valid and persisted in the + # database. Otherwise, the LDAP access check will fail since + # the user doesn't have an associated LDAP identity. + super && allowed? end def ldap_config diff --git a/lib/gitlab/background_migration/backfill_ci_runner_semver.rb b/lib/gitlab/background_migration/backfill_ci_runner_semver.rb new file mode 100644 index 00000000000..0901649f789 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_ci_runner_semver.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to update semver column in ci_runners in batches based on existing version values + class BackfillCiRunnerSemver < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + each_sub_batch( + operation_name: :backfill_ci_runner_semver, + batching_scope: ->(relation) { relation.where('semver::cidr IS NULL') } + ) do |sub_batch| + ranged_query = sub_batch.select( + %q(id AS r_id, + substring(ci_runners.version FROM 'v?(\d+\.\d+\.\d+)') AS extracted_semver) + ) + + update_sql = <<~SQL + UPDATE + ci_runners + SET semver = extracted_semver + FROM (#{ranged_query.to_sql}) v + WHERE id = v.r_id + AND v.extracted_semver IS NOT NULL + SQL + + connection.execute(update_sql) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb new file mode 100644 index 00000000000..b2d38ce6aa4 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + # Rechedules the backfill for the `issue_search_data` table for issues imported prior + # to the fix for the imported issues search data bug: + + class BackfillImportedIssueSearchData < BatchedMigrationJob + SUB_BATCH_SIZE = 1_000 + + def perform + each_sub_batch( + operation_name: :update_search_data + ) do |sub_batch| + update_search_data(sub_batch) + rescue ActiveRecord::StatementInvalid => e + raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') + + update_search_data_individually(sub_batch) + end + end + + private + + def update_search_data(relation) + ApplicationRecord.connection.execute( + <<~SQL + INSERT INTO issue_search_data + SELECT + project_id, + id, + NOW(), + NOW(), + setweight(to_tsvector('english', LEFT(title, 255)), 'A') || setweight(to_tsvector('english', LEFT(REGEXP_REPLACE(description, '[A-Za-z0-9+/@]{50,}', ' ', 'g'), 1048576)), 'B') + FROM (#{relation.limit(SUB_BATCH_SIZE).to_sql}) issues + ON CONFLICT DO NOTHING + SQL + ) + end + + def update_search_data_individually(relation) + relation.pluck(:id).each do |issue_id| + update_search_data(relation.klass.where(id: issue_id)) + sleep(pause_ms * 0.001) + rescue ActiveRecord::StatementInvalid => e + raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') + + logger.error( + message: "Error updating search data: #{e.message}", + class: relation.klass.name, + model_id: issue_id + ) + end + end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb b/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb new file mode 100644 index 00000000000..ec813022b8f --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_statistics_container_repository_size.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fill container_registry_size for project_statistics + class BackfillProjectStatisticsContainerRepositorySize < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform + # no-op + end + end + end +end + +Gitlab::BackgroundMigration::BackfillProjectStatisticsContainerRepositorySize.prepend_mod_with('Gitlab::BackgroundMigration::BackfillProjectStatisticsContainerRepositorySize') # rubocop:disable Layout/LineLength diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 442eab0673e..c47b1735ccf 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -44,7 +44,19 @@ module Gitlab end end - def parent_batch_relation(batching_scope) + def distinct_each_batch(operation_name: :default, batching_arguments: {}) + all_batching_arguments = { column: batch_column, of: sub_batch_size }.merge(batching_arguments) + + parent_batch_relation.distinct_each_batch(**all_batching_arguments) do |relation| + batch_metrics.instrument_operation(operation_name) do + yield relation + end + + sleep([pause_ms, 0].max * 0.001) + end + end + + def parent_batch_relation(batching_scope = nil) parent_relation = define_batchable_model(batch_table, connection: connection) .where(batch_column => start_id..end_id) diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb index 06036eebcb9..7d5fef67c25 100644 --- a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb @@ -8,7 +8,7 @@ module Gitlab # # If no more batches exist in the table, returns nil. class BackfillIssueWorkItemTypeBatchingStrategy < PrimaryKeyBatchingStrategy - def apply_additional_filters(relation, job_arguments:) + def apply_additional_filters(relation, job_arguments:, job_class: nil) issue_type = job_arguments.first relation.where(issue_type: issue_type) diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_project_namespace_per_group_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_project_namespace_per_group_batching_strategy.rb index f352c527b54..68be42dc0a0 100644 --- a/lib/gitlab/background_migration/batching_strategies/backfill_project_namespace_per_group_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/backfill_project_namespace_per_group_batching_strategy.rb @@ -16,7 +16,7 @@ module Gitlab # batch_min_value - The minimum value which the next batch will start at # batch_size - The size of the next batch # job_arguments - The migration job arguments - def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:) + def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:, job_class: nil) next_batch_bounds = nil model_class = ::Gitlab::BackgroundMigration::ProjectNamespaces::Models::Project quoted_column_name = model_class.connection.quote_column_name(column_name) diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb new file mode 100644 index 00000000000..9ad119310f7 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Batching class to use for back-filling project_statistic's container_registry_size. + # Batches will be scoped to records where the project_ids are migrated + # + # If no more batches exist in the table, returns nil. + class BackfillProjectStatisticsWithContainerRegistrySizeBatchingStrategy < PrimaryKeyBatchingStrategy + MIGRATION_PHASE_1_ENDED_AT = Date.new(2022, 01, 23).freeze + + def apply_additional_filters(relation, job_arguments: [], job_class: nil) + relation.where(created_at: MIGRATION_PHASE_1_ENDED_AT..).or( + relation.where(migration_state: 'import_done') + ).select(:project_id).distinct + end + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb b/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb new file mode 100644 index 00000000000..e1855b6cfee --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Batching class to use for setting state in vulnerabilitites table. + # Batches will be scoped to records where the dismissed_at is set. + # + # If no more batches exist in the table, returns nil. + class DismissedVulnerabilitiesStrategy < PrimaryKeyBatchingStrategy + def apply_additional_filters(relation, job_arguments: [], job_class: nil) + relation.where.not(dismissed_at: nil) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy.rb new file mode 100644 index 00000000000..5cad9d2e3c4 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/loose_index_scan_batching_strategy.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # This strategy provides an efficient way to iterate over columns with non-distinct values. + # A common use case would be iterating over a foreign key columns, for example issues.project_id + class LooseIndexScanBatchingStrategy < BaseStrategy + include Gitlab::Database::DynamicModelHelpers + + # Finds and returns the next batch in the table. + # + # table_name - The table to batch over + # column_name - The column to batch over + # batch_min_value - The minimum value which the next batch will start at + # batch_size - The size of the next batch + # job_arguments - The migration job arguments + # job_class - The migration job class + def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:, job_class: nil) + model_class = define_batchable_model(table_name, connection: connection) + + quoted_column_name = model_class.connection.quote_column_name(column_name) + relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) + next_batch_bounds = nil + + relation.distinct_each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop + next_batch_bounds = batch.pluck(Arel.sql("MIN(#{quoted_column_name}), MAX(#{quoted_column_name})")).first + + break + end + + next_batch_bounds + end + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb index e7a68b183b8..c2f59bf9c76 100644 --- a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb @@ -18,12 +18,13 @@ module Gitlab # batch_min_value - The minimum value which the next batch will start at # batch_size - The size of the next batch # job_arguments - The migration job arguments - def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:) + # job_class - The migration job class + def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:, job_class: nil) model_class = define_batchable_model(table_name, connection: connection) quoted_column_name = model_class.connection.quote_column_name(column_name) relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) - relation = apply_additional_filters(relation, job_arguments: job_arguments) + relation = apply_additional_filters(relation, job_arguments: job_arguments, job_class: job_class) next_batch_bounds = nil relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop @@ -35,19 +36,11 @@ module Gitlab next_batch_bounds end - # Strategies based on PrimaryKeyBatchingStrategy can use - # this method to easily apply additional filters. - # - # Example: - # - # class MatchingType < PrimaryKeyBatchingStrategy - # def apply_additional_filters(relation, job_arguments:) - # type = job_arguments.first - # - # relation.where(type: type) - # end - # end - def apply_additional_filters(relation, job_arguments: []) + def apply_additional_filters(relation, job_arguments: [], job_class: nil) + if job_class.respond_to?(:batching_scope) + return job_class.batching_scope(relation, job_arguments: job_arguments) + end + relation end end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb new file mode 100644 index 00000000000..e759d504f8d --- /dev/null +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for inactive, public projects + class DisableLegacyOpenSourceLicenseForInactivePublicProjects < + ::Gitlab::BackgroundMigration::BatchedMigrationJob + PUBLIC = 20 + LAST_ACTIVITY_DATE = '2021-07-01' + + # Migration only version of `project_settings` table + class ProjectSetting < ApplicationRecord + self.table_name = 'project_settings' + end + + def perform + each_sub_batch( + operation_name: :disable_legacy_open_source_license_available, + batching_scope: ->(relation) { + relation.where(visibility_level: PUBLIC).where('last_activity_at < ?', LAST_ACTIVITY_DATE) + } + ) do |sub_batch| + ProjectSetting.where(project_id: sub_batch).update_all(legacy_open_source_license_available: false) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb index a34e923545c..914ababa5c2 100644 --- a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb +++ b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb @@ -68,7 +68,7 @@ module Gitlab def valid_json?(metadata) Oj.load(metadata) true - rescue Oj::ParseError, Encoding::UndefinedConversionError + rescue Oj::ParseError, EncodingError, JSON::ParserError, Encoding::UndefinedConversionError false end diff --git a/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb b/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb new file mode 100644 index 00000000000..3f04e04fc4d --- /dev/null +++ b/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates the value operations_access_level to the new colums + # monitor_access_level, deployments_access_level, infrastructure_access_level. + # The operations_access_level setting is being split into three seperate toggles. + class PopulateOperationVisibilityPermissionsFromOperations < BatchedMigrationJob + def perform + each_sub_batch(operation_name: :populate_operations_visibility) do |batch| + batch.update_all('monitor_access_level=operations_access_level,' \ + 'infrastructure_access_level=operations_access_level,' \ + ' feature_flags_access_level=operations_access_level,'\ + ' environments_access_level=operations_access_level') + end + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'PopulateOperationVisibilityPermissionsFromOperations', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index db7afd59f4d..72380af2c53 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -79,10 +79,6 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # r # rubocop: disable Metrics/AbcSize,Metrics/MethodLength,Metrics/BlockLength def perform(start_id, end_id) - unless Feature.enabled?(:migrate_vulnerability_finding_uuids) - return log_info('Migration is disabled by the feature flag', start_id: start_id, end_id: end_id) - end - log_info('Migration started', start_id: start_id, end_id: end_id) VulnerabilitiesFinding diff --git a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb new file mode 100644 index 00000000000..fd6cbcb8d05 --- /dev/null +++ b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Change the vulnerability state to `dismissed` if `dismissed_at` field is not null + class SetCorrectVulnerabilityState < BatchedMigrationJob + DISMISSED_STATE = 2 + + def perform + each_sub_batch( + operation_name: :update_vulnerabilities_state, + batching_scope: -> (relation) { relation.where.not(dismissed_at: nil) } + ) do |sub_batch| + sub_batch.update_all(state: DISMISSED_STATE) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb b/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb new file mode 100644 index 00000000000..04a2ceebef8 --- /dev/null +++ b/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class is used to update the delayed_project_removal column + # for user namespaces of the namespace_settings table. + class UpdateDelayedProjectRemovalToNullForUserNamespaces < Gitlab::BackgroundMigration::BatchedMigrationJob + # Migration only version of `namespace_settings` table + class NamespaceSetting < ::ApplicationRecord + self.table_name = 'namespace_settings' + end + + def perform + each_sub_batch( + operation_name: :set_delayed_project_removal_to_null_for_user_namespace + ) do |sub_batch| + set_delayed_project_removal_to_null_for_user_namespace(sub_batch) + end + end + + private + + def set_delayed_project_removal_to_null_for_user_namespace(relation) + NamespaceSetting.connection.execute( + <<~SQL + UPDATE namespace_settings + SET delayed_project_removal = NULL + WHERE + namespace_settings.namespace_id IN ( + SELECT + namespace_settings.namespace_id + FROM + namespace_settings + INNER JOIN namespaces ON namespaces.id = namespace_settings.namespace_id + WHERE + namespaces.id IN (#{relation.select(:namespace_id).to_sql}) + AND namespaces.type = 'User' + AND namespace_settings.delayed_project_removal = FALSE) + SQL + ) + end + end + end +end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index f79baadb8ea..d58de7eb211 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -86,13 +86,15 @@ module Gitlab create_labels + issue_type_id = WorkItems::Type.default_issue_type.id + client.issues(repo).each do |issue| - import_issue(issue) + import_issue(issue, issue_type_id) end end # rubocop: disable CodeReuse/ActiveRecord - def import_issue(issue) + def import_issue(issue, issue_type_id) description = '' description += @formatter.author_line(issue.author) unless find_user_id(issue.author) description += issue.description @@ -106,7 +108,9 @@ module Gitlab description: description, state_id: Issue.available_states[issue.state], author_id: gitlab_user_id(project, issue.author), + namespace_id: project.project_namespace_id, milestone: milestone, + work_item_type_id: issue_type_id, created_at: issue.created_at, updated_at: issue.updated_at ) diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb index 9cb3d71f5c3..8fcc03ec437 100644 --- a/lib/gitlab/changelog/config.rb +++ b/lib/gitlab/changelog/config.rb @@ -7,9 +7,9 @@ module Gitlab # When rendering changelog entries, authors are not included. AUTHORS_NONE = 'none' - # The path to the configuration file as stored in the project's Git + # The default path to the configuration file as stored in the project's Git # repository. - FILE_PATH = '.gitlab/changelog_config.yml' + DEFAULT_FILE_PATH = '.gitlab/changelog_config.yml' # The default date format to use for formatting release dates. DEFAULT_DATE_FORMAT = '%Y-%m-%d' @@ -36,8 +36,9 @@ module Gitlab attr_accessor :date_format, :categories, :template, :tag_regex, :always_credit_user_ids - def self.from_git(project, user = nil) - if (yaml = project.repository.changelog_config.presence) + def self.from_git(project, user = nil, path = nil) + yaml = project.repository.changelog_config('HEAD', path.presence || DEFAULT_FILE_PATH) + if yaml.present? from_hash(project, YAML.safe_load(yaml), user) else new(project) diff --git a/lib/gitlab/ci/build/artifacts/expire_in_parser.rb b/lib/gitlab/ci/build/artifacts/expire_in_parser.rb deleted file mode 100644 index 848208c5cdd..00000000000 --- a/lib/gitlab/ci/build/artifacts/expire_in_parser.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Build - module Artifacts - class ExpireInParser - def self.validate_duration(value) - new(value).validate_duration - end - - def initialize(value) - @value = value - end - - def validate_duration - return true if never? - - cached_parse - end - - def seconds_from_now - parse&.seconds&.from_now - end - - private - - attr_reader :value - - def cached_parse - return validation_cache[value] if validation_cache.key?(value) - - validation_cache[value] = safe_parse - end - - def safe_parse - parse - rescue ChronicDuration::DurationParseError - false - end - - def parse - return if never? - - ChronicDuration.parse(value) - end - - def validation_cache - Gitlab::SafeRequestStore[:ci_expire_in_parser_cache] ||= {} - end - - def never? - value.to_s.casecmp('never') == 0 - end - end - end - end - end -end diff --git a/lib/gitlab/ci/build/duration_parser.rb b/lib/gitlab/ci/build/duration_parser.rb new file mode 100644 index 00000000000..9385dccd5f3 --- /dev/null +++ b/lib/gitlab/ci/build/duration_parser.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class DurationParser + def self.validate_duration(value) + new(value).validate_duration + end + + def initialize(value) + @value = value + end + + def validate_duration + return true if never? + + cached_parse + end + + def seconds_from_now + parse&.seconds&.from_now + end + + private + + attr_reader :value + + def cached_parse + return validation_cache[value] if validation_cache.key?(value) + + validation_cache[value] = safe_parse + end + + def safe_parse + parse + rescue ChronicDuration::DurationParseError + false + end + + def parse + return if never? + + ChronicDuration.parse(value) + end + + def validation_cache + Gitlab::SafeRequestStore[:ci_expire_in_parser_cache] ||= {} + end + + def never? + value.to_s.casecmp('never') == 0 + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb index 4c5f02b4f7b..1bcd87c9d93 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb @@ -4,8 +4,10 @@ module Gitlab module Ci module Build class Rules::Rule::Clause::Changes < Rules::Rule::Clause + include Gitlab::Utils::StrongMemoize + def initialize(globs) - @globs = Array(globs) + @globs = globs end def satisfied_by?(pipeline, context) @@ -19,13 +21,25 @@ module Gitlab end end + private + def expand_globs(context) - return @globs unless context + return paths unless context - @globs.map do |glob| + paths.map do |glob| ExpandVariables.expand_existing(glob, -> { context.variables_hash }) end end + + def paths + strong_memoize(:paths) do + if @globs.is_a?(Array) + @globs + else + Array(@globs[:paths]) + end + end + end end end end diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 56eeb5eeb06..3b0cbc6b69e 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -42,7 +42,7 @@ module Gitlab inclusion: { in: %w[on_success on_failure always], message: 'should be on_success, on_failure ' \ 'or always' } - validates :expire_in, duration: { parser: ::Gitlab::Ci::Build::Artifacts::ExpireInParser } + validates :expire_in, duration: { parser: ::Gitlab::Ci::Build::DurationParser } end end diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index bc39abfe977..96ba3553b46 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -54,7 +54,7 @@ module Gitlab validates :on_stop, type: String, allow_nil: true validates :kubernetes, type: Hash, allow_nil: true - validates :auto_stop_in, duration: true, allow_nil: true + validates :auto_stop_in, duration: { parser: ::Gitlab::Ci::Build::DurationParser }, allow_nil: true end end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 79443f69b03..96ac959a3f4 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -48,7 +48,7 @@ module Gitlab { name: @config[:name], entrypoint: @config[:entrypoint], - ports: ports_value, + ports: (ports_value if ports_defined?), pull_policy: (ci_docker_image_pull_policy_enabled? ? pull_policy_value : nil) }.compact else diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 46afedbcc3a..78794f524f4 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -71,9 +71,9 @@ module Gitlab end def compose!(deps = nil) - super do - has_workflow_rules = deps&.workflow_entry&.has_rules? + has_workflow_rules = deps&.workflow_entry&.has_rules? + super do # If workflow:rules: or rules: are used # they are considered not compatible # with `only/except` defaults @@ -86,12 +86,10 @@ module Gitlab @entries.delete(:except) unless except_defined? # rubocop:disable Gitlab/ModuleWithInstanceVariables end - unless has_workflow_rules - validate_against_warnings - end - yield if block_given? end + + validate_against_warnings unless has_workflow_rules end def validate_against_warnings diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb index 53e52981471..91be1bb3ee4 100644 --- a/lib/gitlab/ci/config/entry/rules.rb +++ b/lib/gitlab/ci/config/entry/rules.rb @@ -13,7 +13,7 @@ module Gitlab end def value - [@config].flatten + [super].flatten end def composable_class diff --git a/lib/gitlab/ci/config/entry/rules/rule/changes.rb b/lib/gitlab/ci/config/entry/rules/rule/changes.rb index be57e089f34..a56b928450a 100644 --- a/lib/gitlab/ci/config/entry/rules/rule/changes.rb +++ b/lib/gitlab/ci/config/entry/rules/rule/changes.rb @@ -6,13 +6,51 @@ module Gitlab module Entry class Rules class Rule - class Changes < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable + class Changes < ::Gitlab::Config::Entry::Simplifiable + strategy :SimpleChanges, if: -> (config) { config.is_a?(Array) } + strategy :ComplexChanges, if: -> (config) { config.is_a?(Hash) } - validations do - validates :config, - array_of_strings: true, - length: { maximum: 50, too_long: "has too many entries (maximum %{count})" } + class SimpleChanges < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, + array_of_strings: true, + length: { maximum: 50, too_long: "has too many entries (maximum %{count})" } + end + + def value + { + paths: config + }.compact + end + end + + class ComplexChanges < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[paths].freeze + REQUIRED_KEYS = %i[paths].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + validates :config, required_keys: REQUIRED_KEYS + + with_options allow_nil: false do + validates :paths, + array_of_strings: true, + length: { maximum: 50, too_long: "has too many entries (maximum %{count})" } + end + end + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["#{location} should be an array or a hash"] + end end end end diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index f27dca4986e..1a35f7de6cf 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -15,11 +15,13 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[name entrypoint command alias ports variables].freeze + ALLOWED_KEYS = %i[name entrypoint command alias ports variables pull_policy].freeze + LEGACY_ALLOWED_KEYS = %i[name entrypoint command alias ports variables].freeze validations do validates :config, hash_or_string: true - validates :config, allowed_keys: ALLOWED_KEYS + validates :config, allowed_keys: ALLOWED_KEYS, if: :ci_docker_image_pull_policy_enabled? + validates :config, allowed_keys: LEGACY_ALLOWED_KEYS, unless: :ci_docker_image_pull_policy_enabled? validates :config, disallowed_keys: %i[ports], unless: :with_image_ports? validates :name, type: String, presence: true validates :entrypoint, array_of_strings: true, allow_nil: true @@ -32,11 +34,14 @@ module Gitlab entry :ports, Entry::Ports, description: 'Ports used to expose the service' + entry :pull_policy, Entry::PullPolicy, + description: 'Pull policy for the service' + entry :variables, ::Gitlab::Ci::Config::Entry::Variables, description: 'Environment variables available for this service.', inherit: false - attributes :ports + attributes :ports, :pull_policy, :variables def alias value[:alias] @@ -55,16 +60,25 @@ module Gitlab end def value - return { name: @config } if string? - return @config if hash? - - {} + if string? + { name: @config } + elsif hash? + @config.merge( + pull_policy: (pull_policy_value if ci_docker_image_pull_policy_enabled?) + ).compact + else + {} + end end def with_image_ports? opt(:with_image_ports) end + def ci_docker_image_pull_policy_enabled? + ::Feature.enabled?(:ci_docker_image_pull_policy) + end + def skip_config_hash_validation? true end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 2def565bc19..ec628399785 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -9,14 +9,20 @@ module Gitlab TimeoutError = Class.new(StandardError) + MAX_INCLUDES = 100 + TRIAL_MAX_INCLUDES = 250 + include ::Gitlab::Utils::StrongMemoize attr_reader :project, :sha, :user, :parent_pipeline, :variables - attr_reader :expandset, :execution_deadline, :logger + attr_reader :expandset, :execution_deadline, :logger, :max_includes delegate :instrument, to: :logger - def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, logger: nil) + def initialize( + project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, + logger: nil + ) @project = project @sha = sha @user = user @@ -25,7 +31,7 @@ module Gitlab @expandset = Set.new @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) - + @max_includes = Feature.enabled?(:ci_increase_includes_to_250, project) ? TRIAL_MAX_INCLUDES : MAX_INCLUDES yield self if block_given? end @@ -52,6 +58,7 @@ module Gitlab ctx.expandset = expandset ctx.execution_deadline = execution_deadline ctx.logger = logger + ctx.max_includes = max_includes end end @@ -86,7 +93,7 @@ module Gitlab protected - attr_writer :expandset, :execution_deadline, :logger + attr_writer :expandset, :execution_deadline, :logger, :max_includes private diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index b7fef081269..89418bd6a21 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -13,7 +13,7 @@ module Gitlab def initialize(params, context) @location = params[:file] - @project_name = params[:project] + @project_name = get_project_name(params[:project]) @ref_name = params[:ref] || 'HEAD' super @@ -122,6 +122,16 @@ module Gitlab ) end end + + # TODO: To be removed after we deprecate usage of array in `project` keyword. + # https://gitlab.com/gitlab-org/gitlab/-/issues/365975 + def get_project_name(project_name) + if project_name.is_a?(Array) + project_name.first + else + project_name + end + end end end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index c1250c82750..2a1060a6059 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -7,8 +7,6 @@ module Gitlab class Mapper include Gitlab::Utils::StrongMemoize - MAX_INCLUDES = 100 - FILE_CLASSES = [ External::File::Remote, External::File::Template, @@ -134,8 +132,8 @@ module Gitlab end def verify_max_includes! - if expandset.count >= MAX_INCLUDES - raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!" + if expandset.count >= context.max_includes + raise TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!" end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index 19678def666..c294291e538 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -64,7 +64,8 @@ module Gitlab if environment.present? fields.merge!( environment: environment.name, - environment_protected: environment_protected?.to_s + environment_protected: environment_protected?.to_s, + deployment_tier: build.environment_deployment_tier || environment.tier ) end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 71dfc1a676c..207b4b5ff8b 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -11,10 +11,10 @@ module Gitlab def perform! logger.instrument_with_sql(:pipeline_save) do BulkInsertableAssociations.with_bulk_insert do - with_bulk_insert_tags do + ::Ci::BulkInsertableTags.with_bulk_insert_tags do pipeline.transaction do pipeline.save! - CommitStatus.bulk_insert_tags!(statuses) + Gitlab::Ci::Tags::BulkInsert.bulk_insert_tags!(statuses) end end end @@ -29,14 +29,6 @@ module Gitlab private - def with_bulk_insert_tags - previous = Thread.current['ci_bulk_insert_tags'] - Thread.current['ci_bulk_insert_tags'] = true - yield - ensure - Thread.current['ci_bulk_insert_tags'] = previous - end - def statuses strong_memoize(:statuses) do pipeline diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 33b9ac9b641..c3e0f043b44 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -42,6 +42,15 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end + def self.pipeline_age_histogram + name = :gitlab_ci_pipeline_age_minutes + comment = 'Pipeline age histogram' + buckets = [5, 30, 120, 720, 1440, 7200, 21600, 43200, 86400, 172800, 518400, 1036800] + # 5m 30m 2h 12h 24h 5d 15d 30d 60d 180d 360d 2y + + ::Gitlab::Metrics.histogram(name, comment, {}, buckets) + end + def self.active_jobs_histogram name = :gitlab_ci_active_jobs comment = 'Total amount of active jobs' diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 7d8303214a5..5cee73238ca 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -250,11 +250,7 @@ module Gitlab end def running_jobs_relation(job) - if ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data) - ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) - else - job.project.builds.running.where(runner: ::Ci::Runner.instance_type) - end + ::Ci::RunningBuild.instance_type.where(project_id: job.project_id) end # rubocop: enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/ci/reports/coverage_report_generator.rb b/lib/gitlab/ci/reports/coverage_report_generator.rb index fd73ed6fd25..76992a48b0a 100644 --- a/lib/gitlab/ci/reports/coverage_report_generator.rb +++ b/lib/gitlab/ci/reports/coverage_report_generator.rb @@ -35,17 +35,7 @@ module Gitlab private def report_builds - if child_pipeline_feature_enabled? - @pipeline.latest_report_builds_in_self_and_descendants(::Ci::JobArtifact.coverage_reports) - else - @pipeline.latest_report_builds(::Ci::JobArtifact.coverage_reports) - end - end - - def child_pipeline_feature_enabled? - strong_memoize(:feature_enabled) do - Feature.enabled?(:ci_child_pipeline_coverage_reports, @pipeline.project) - end + @pipeline.latest_report_builds_in_self_and_descendants(::Ci::JobArtifact.coverage_reports) end end end diff --git a/lib/gitlab/ci/reports/test_reports.rb b/lib/gitlab/ci/reports/test_report.rb index a5a630642e5..4fc10dd736e 100644 --- a/lib/gitlab/ci/reports/test_reports.rb +++ b/lib/gitlab/ci/reports/test_report.rb @@ -3,7 +3,7 @@ module Gitlab module Ci module Reports - class TestReports + class TestReport attr_reader :test_suites def initialize diff --git a/lib/gitlab/ci/reports/test_reports_comparer.rb b/lib/gitlab/ci/reports/test_reports_comparer.rb index c6f17f0764f..497831ae5a7 100644 --- a/lib/gitlab/ci/reports/test_reports_comparer.rb +++ b/lib/gitlab/ci/reports/test_reports_comparer.rb @@ -9,7 +9,7 @@ module Gitlab attr_reader :base_reports, :head_reports def initialize(base_reports, head_reports) - @base_reports = base_reports || TestReports.new + @base_reports = base_reports || TestReport.new @head_reports = head_reports end diff --git a/lib/gitlab/ci/runner/metrics.rb b/lib/gitlab/ci/runner/metrics.rb new file mode 100644 index 00000000000..8df126decff --- /dev/null +++ b/lib/gitlab/ci/runner/metrics.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Runner + class Metrics + extend Gitlab::Utils::StrongMemoize + + def increment_runner_authentication_success_counter(runner_type: 'unknown_type') + raise ArgumentError, "unknown runner type: #{runner_type}" unless + ::Ci::Runner.runner_types.include? runner_type + + self.class.runner_authentication_success_counter.increment(runner_type: runner_type) + end + + def increment_runner_authentication_failure_counter + self.class.runner_authentication_failure_counter.increment + end + + def self.runner_authentication_success_counter + strong_memoize(:runner_authentication_success) do + name = :gitlab_ci_runner_authentication_success_total + comment = 'Runner authentication success' + labels = { runner_type: nil } + + ::Gitlab::Metrics.counter(name, comment, labels) + end + end + + def self.runner_authentication_failure_counter + strong_memoize(:runner_authentication_failure) do + name = :gitlab_ci_runner_authentication_failure_total + comment = 'Runner authentication failure' + + ::Gitlab::Metrics.counter(name, comment) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/runner_releases.rb b/lib/gitlab/ci/runner_releases.rb index 944c24ca128..8773ecbf09e 100644 --- a/lib/gitlab/ci/runner_releases.rb +++ b/lib/gitlab/ci/runner_releases.rb @@ -6,48 +6,83 @@ module Gitlab include Singleton RELEASES_VALIDITY_PERIOD = 1.day - RELEASES_VALIDITY_AFTER_ERROR_PERIOD = 5.seconds INITIAL_BACKOFF = 5.seconds MAX_BACKOFF = 1.hour BACKOFF_GROWTH_FACTOR = 2.0 def initialize - reset! + reset_backoff! end # Returns a sorted list of the publicly available GitLab Runner releases # def releases - return @releases unless Time.now.utc >= @expire_time + return if backoff_active? + + Rails.cache.fetch( + cache_key, + skip_nil: true, + expires_in: RELEASES_VALIDITY_PERIOD, + race_condition_ttl: 10.seconds + ) do + response = Gitlab::HTTP.try_get(runner_releases_url) + @releases_by_minor = nil + + unless response&.success? + @backoff_expire_time = next_backoff.from_now + break nil + end + + reset_backoff! + extract_releases(response) + end + end + + # Returns a hash with the latest runner version per minor release + # + def releases_by_minor + return unless releases - @releases = fetch_new_releases + @releases_by_minor ||= releases.group_by(&:without_patch).transform_values(&:max) end - def reset! - @expire_time = Time.now.utc - @releases = nil + def reset_backoff! + @backoff_expire_time = nil @backoff_count = 0 end - public_class_method :instance - private - def fetch_new_releases - response = Gitlab::HTTP.try_get(::Gitlab::CurrentSettings.current_application_settings.public_runner_releases_url) + def runner_releases_url + @runner_releases_url ||= ::Gitlab::CurrentSettings.current_application_settings.public_runner_releases_url + end - releases = response.success? ? extract_releases(response) : nil - ensure - @expire_time = (releases ? RELEASES_VALIDITY_PERIOD : next_backoff).from_now + def cache_key + runner_releases_url + end + + def backoff_active? + return false unless @backoff_expire_time + + Time.now.utc < @backoff_expire_time end def extract_releases(response) - response.parsed_response.map { |release| parse_runner_release(release) }.sort! + return unless response.parsed_response.is_a?(Array) + + releases = response.parsed_response + .map { |release| parse_runner_release(release) } + .select(&:valid?) + .sort! + + return if releases.empty? && response.parsed_response.present? + + releases end def parse_runner_release(release) - ::Gitlab::VersionInfo.parse(release['name'].delete_prefix('v')) + ::Gitlab::VersionInfo.parse(release['name'], parse_suffix: true) end def next_backoff diff --git a/lib/gitlab/ci/runner_upgrade_check.rb b/lib/gitlab/ci/runner_upgrade_check.rb index 0808290fe5b..10a89bb15d4 100644 --- a/lib/gitlab/ci/runner_upgrade_check.rb +++ b/lib/gitlab/ci/runner_upgrade_check.rb @@ -5,76 +5,71 @@ module Gitlab class RunnerUpgradeCheck include Singleton - STATUSES = { - invalid: 'Runner version is not valid.', - not_available: 'Upgrade is not available for the runner.', - available: 'Upgrade is available for the runner.', - recommended: 'Upgrade is available and recommended for the runner.' - }.freeze - - def initialize - reset! - end - def check_runner_upgrade_status(runner_version) - return :invalid unless runner_version + runner_version = ::Gitlab::VersionInfo.parse(runner_version, parse_suffix: true) - releases = RunnerReleases.instance.releases - orig_runner_version = runner_version - runner_version = ::Gitlab::VersionInfo.parse(runner_version) unless runner_version.is_a?(::Gitlab::VersionInfo) + return { invalid_version: runner_version } unless runner_version.valid? + return { error: runner_version } unless runner_releases_store.releases - raise ArgumentError, "'#{orig_runner_version}' is not a valid version" unless runner_version.valid? + # Recommend update if outside of backport window + recommended_version = recommendation_if_outside_backport_window(runner_version) + return { recommended: recommended_version } if recommended_version - gitlab_minor_version = version_without_patch(@gitlab_version) + # Recommend patch update if there's a newer release in a same minor branch as runner + recommended_version = recommended_runner_release_update(runner_version) + return { recommended: recommended_version } if recommended_version - available_releases = releases - .reject { |release| release.major > @gitlab_version.major } - .reject do |release| - release_minor_version = version_without_patch(release) + # Consider update if there's a newer release within the currently deployed GitLab version + available_version = available_runner_release(runner_version) + return { available: available_version } if available_version - # Do not reject a patch update, even if the runner is ahead of the instance version - next false if version_without_patch(runner_version) == release_minor_version + { not_available: runner_version } + end - release_minor_version > gitlab_minor_version - end + private - return :recommended if available_releases.any? { |available_rel| patch_update?(available_rel, runner_version) } - return :recommended if outside_backport_window?(runner_version, releases) - return :available if available_releases.any? { |available_rel| available_rel > runner_version } + def recommended_runner_release_update(runner_version) + recommended_release = runner_releases_store.releases_by_minor[runner_version.without_patch] + return recommended_release if recommended_release && recommended_release > runner_version - :not_available + # Consider the edge case of pre-release runner versions that get registered, but are never published. + # In this case, suggest the latest compatible runner version + latest_release = runner_releases_store.releases_by_minor.values.select { |v| v < gitlab_version }.max + latest_release if latest_release && latest_release > runner_version end - def reset! - @gitlab_version = ::Gitlab::VersionInfo.parse(::Gitlab::VERSION) + def available_runner_release(runner_version) + available_release = runner_releases_store.releases_by_minor[gitlab_version.without_patch] + available_release if available_release && available_release > runner_version end - public_class_method :instance - - private - - def patch_update?(available_release, runner_version) - # https://docs.gitlab.com/ee/policy/maintenance.html#patch-releases - available_release.major == runner_version.major && - available_release.minor == runner_version.minor && - available_release.patch > runner_version.patch + def gitlab_version + @gitlab_version ||= ::Gitlab::VersionInfo.parse(::Gitlab::VERSION, parse_suffix: true) end - def outside_backport_window?(runner_version, releases) - return false if runner_version >= releases.last # return early if runner version is too new - - latest_minor_releases = releases.map { |r| version_without_patch(r) }.uniq { |v| v.to_s } - latest_version_position = latest_minor_releases.count - 1 - runner_version_position = latest_minor_releases.index(version_without_patch(runner_version)) - - return true if runner_version_position.nil? # consider outside if version is too old - - # https://docs.gitlab.com/ee/policy/maintenance.html#backporting-to-older-releases - latest_version_position - runner_version_position > 2 + def runner_releases_store + RunnerReleases.instance end - def version_without_patch(version) - ::Gitlab::VersionInfo.new(version.major, version.minor, 0) + def recommendation_if_outside_backport_window(runner_version) + return if runner_releases_store.releases.empty? + return if runner_version >= runner_releases_store.releases.last # return early if runner version is too new + + minor_releases_with_index = runner_releases_store.releases_by_minor.keys.each_with_index.to_h + runner_minor_version_index = minor_releases_with_index[runner_version.without_patch] + if runner_minor_version_index + # https://docs.gitlab.com/ee/policy/maintenance.html#backporting-to-older-releases + outside_window = minor_releases_with_index.count - runner_minor_version_index > 3 + + if outside_window + recommended_release = runner_releases_store.releases_by_minor[gitlab_version.without_patch] + + recommended_release if recommended_release && recommended_release > runner_version + end + else + # If unknown runner version, then recommend the latest version for the GitLab instance + recommended_runner_release_update(gitlab_version) + end end end end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 1a074c1af53..5d60aa8f540 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -20,10 +20,13 @@ module Gitlab scheduler_failure: 'scheduler failure', data_integrity_failure: 'data integrity failure', forward_deployment_failure: 'forward deployment failure', + protected_environment_failure: 'protected environment failure', pipeline_loop_detected: 'job would create infinitely looping pipelines', invalid_bridge_trigger: 'downstream pipeline trigger definition is invalid', downstream_bridge_project_not_found: 'downstream project could not be found', + upstream_bridge_project_not_found: 'upstream project could not be found', insufficient_bridge_permissions: 'no permissions to trigger downstream pipeline', + insufficient_upstream_permissions: 'no permissions to read upstream project', bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline', downstream_pipeline_creation_failed: 'downstream pipeline can not be created', secrets_provider_not_found: 'secrets provider can not be found', @@ -75,5 +78,3 @@ module Gitlab end end end - -Gitlab::Ci::Status::Build::Failed.prepend_mod_with('Gitlab::Ci::Status::Build::Failed') diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 3b2da773102..e854164d377 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -7,10 +7,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize # This class accepts an array of arrays/hashes/or objects - # - # The parameter `project` is only used for the feature flag check, and will be removed with - # https://gitlab.com/gitlab-org/gitlab/-/issues/321972 - def initialize(all_statuses, with_allow_failure: true, dag: false, project: nil) + def initialize(all_statuses, with_allow_failure: true, dag: false) unless all_statuses.respond_to?(:pluck) raise ArgumentError, "all_statuses needs to respond to `.pluck`" end @@ -19,7 +16,6 @@ module Gitlab @status_key = 0 @allow_failure_key = 1 if with_allow_failure @dag = dag - @project = project consume_all_statuses(all_statuses) end diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb index 29f3731a9b4..2e56e47f5b8 100644 --- a/lib/gitlab/ci/tags/bulk_insert.rb +++ b/lib/gitlab/ci/tags/bulk_insert.rb @@ -9,40 +9,44 @@ module Gitlab TAGGINGS_BATCH_SIZE = 1000 TAGS_BATCH_SIZE = 500 - def initialize(statuses) - @statuses = statuses + def self.bulk_insert_tags!(taggables) + Gitlab::Ci::Tags::BulkInsert.new(taggables).insert! + end + + def initialize(taggables) + @taggables = taggables end def insert! - return false if tag_list_by_status.empty? + return false if tag_list_by_taggable.empty? persist_build_tags! end private - attr_reader :statuses + attr_reader :taggables - def tag_list_by_status - strong_memoize(:tag_list_by_status) do - statuses.each.with_object({}) do |status, acc| - tag_list = status.tag_list + def tag_list_by_taggable + strong_memoize(:tag_list_by_taggable) do + taggables.each.with_object({}) do |taggable, acc| + tag_list = taggable.tag_list next unless tag_list - acc[status] = tag_list + acc[taggable] = tag_list end end end def persist_build_tags! - all_tags = tag_list_by_status.values.flatten.uniq.reject(&:blank?) + all_tags = tag_list_by_taggable.values.flatten.uniq.reject(&:blank?) tag_records_by_name = create_tags(all_tags).index_by(&:name) taggings = build_taggings_attributes(tag_records_by_name) return false if taggings.empty? taggings.each_slice(TAGGINGS_BATCH_SIZE) do |taggings_slice| - ActsAsTaggableOn::Tagging.insert_all!(taggings) + ActsAsTaggableOn::Tagging.insert_all!(taggings_slice) end true @@ -65,24 +69,24 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def build_taggings_attributes(tag_records_by_name) - taggings = statuses.flat_map do |status| - tag_list = tag_list_by_status[status] + taggings = taggables.flat_map do |taggable| + tag_list = tag_list_by_taggable[taggable] next unless tag_list tags = tag_records_by_name.values_at(*tag_list) - taggings_for(tags, status) + taggings_for(tags, taggable) end taggings.compact! taggings end - def taggings_for(tags, status) + def taggings_for(tags, taggable) tags.map do |tag| { tag_id: tag.id, - taggable_type: CommitStatus.name, - taggable_id: status.id, + taggable_type: taggable.class.base_class.name, + taggable_id: taggable.id, created_at: Time.current, context: 'tags' } diff --git a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml index 17e49440784..1ac9c319429 100644 --- a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml @@ -7,6 +7,7 @@ stages: - build - test - review + - dast - deploy - production - cleanup diff --git a/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml index 9f0e9bcc1f2..ee52bc91ab3 100644 --- a/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml @@ -30,24 +30,24 @@ before_script: - apt-get --quiet update --yes - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1 - # Setup path as ANDROID_SDK_ROOT for moving/exporting the downloaded sdk into it - - export ANDROID_SDK_ROOT="${PWD}/android-home" + # Setup path as ANDROID_HOME for moving/exporting the downloaded sdk into it + - export ANDROID_HOME="${PWD}/android-home" # Create a new directory at specified location - - install -d $ANDROID_SDK_ROOT + - install -d $ANDROID_HOME # Here we are installing androidSDK tools from official source, # (the key thing here is the url from where you are downloading these sdk tool for command line, so please do note this url pattern there and here as well) # after that unzipping those tools and # then running a series of SDK manager commands to install necessary android SDK packages that'll allow the app to build - - wget --output-document=$ANDROID_SDK_ROOT/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip - # move to the archive at ANDROID_SDK_ROOT - - pushd $ANDROID_SDK_ROOT + - wget --output-document=$ANDROID_HOME/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip + # move to the archive at ANDROID_HOME + - pushd $ANDROID_HOME - unzip -d cmdline-tools cmdline-tools.zip - pushd cmdline-tools # since commandline tools version 7583922 the root folder is named "cmdline-tools" so we rename it if necessary - mv cmdline-tools tools || true - popd - popd - - export PATH=$PATH:${ANDROID_SDK_ROOT}/cmdline-tools/tools/bin/ + - export PATH=$PATH:${ANDROID_HOME}/cmdline-tools/tools/bin/ # Nothing fancy here, just checking sdkManager version - sdkmanager --version diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index f39a84bceec..004c2897b60 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/C++.gitlab-ci.yml b/lib/gitlab/ci/templates/C++.gitlab-ci.yml index c078c99f352..3096af1b173 100644 --- a/lib/gitlab/ci/templates/C++.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/C++.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml index f166da9bdd6..a64f87193a9 100644 --- a/lib/gitlab/ci/templates/Chef.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Chef.gitlab-ci.yml @@ -1,14 +1,17 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Chef.gitlab-ci.yml - # This template uses Test Kitchen with the kitchen-dokken driver to # perform functional testing. Doing so requires that your runner be a # Docker runner configured for privileged mode. Please see # https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode # for help configuring your runner properly, or, if you want to switch # to a different driver, see http://kitchen.ci/docs/drivers +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Chef.gitlab-ci.yml image: "chef/chefdk" services: diff --git a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml index 0f9e28c9a8e..4fe37ceaeaa 100644 --- a/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Clojure.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml index 8886929646d..68b55b782cd 100644 --- a/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Crystal.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml index 6354db38f58..35401e62fe2 100644 --- a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml index 1eb920c7747..83ddce936e6 100644 --- a/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Elixir.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml index a5c261e367a..021662ab416 100644 --- a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml index 21a599fc78d..464b81965f2 100644 --- a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml - # This is a sample GitLab CI/CD configuration file that should run without any modifications. # It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, # it uses echo commands to simulate the pipeline execution. @@ -11,6 +6,14 @@ # Stages run in sequential order, but jobs within stages run in parallel. # # For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml stages: # List of stages for jobs, and their order of execution - build diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index bd8e1020c4e..603aede4d46 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index 7e59354c4a1..03c8941169f 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Grails.gitlab-ci.yml - # This template uses the java:8 docker image because there isn't any # official Grails image at this moment # @@ -12,6 +7,14 @@ # Feel free to change GRAILS_VERSION version with your project version (3.0.1, 3.1.1,...) # Feel free to change GRADLE_VERSION version with your gradle project version (2.13, 2.14,...) # If you use Angular profile, this yml it's prepared to work with it +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Grails.gitlab-ci.yml image: java:8 diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 6a95d042842..86e3ace84c5 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.26" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.29" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 8f1124373c4..b41e92e3a56 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -4,6 +4,14 @@ variables: .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" +.common_rules: &common_rules + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME + when: never + - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH + when: never + - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given + when: never + dast_environment_deploy: extends: .dast-auto-deploy stage: review @@ -23,12 +31,7 @@ dast_environment_deploy: artifacts: paths: [environment_url.txt] rules: - - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME - when: never - - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH - when: never - - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given - when: never + - *common_rules - if: $CI_COMMIT_BRANCH && ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ @@ -47,13 +50,53 @@ stop_dast_environment: action: stop needs: ["dast"] rules: - - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME - when: never - - if: $DAST_DISABLED || $DAST_DISABLED_FOR_DEFAULT_BRANCH - when: never - - if: $DAST_WEBSITE # we don't need to create a review app if a URL is already given - when: never + - *common_rules - if: $CI_COMMIT_BRANCH && ($CI_KUBERNETES_ACTIVE || $KUBECONFIG) && $GITLAB_FEATURES =~ /\bdast\b/ when: always + +.ecs_image: + image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' + +.ecs_rules: &ecs_rules + - if: $AUTO_DEVOPS_PLATFORM_TARGET != "ECS" + when: never + - if: $CI_KUBERNETES_ACTIVE || $KUBECONFIG + when: never + +dast_ecs_environment_deploy: + extends: .ecs_image + stage: review + script: + - ecs update-task-definition + - echo "http://$(ecs get-task-hostname)" > environment_url.txt + environment: + name: dast-default + on_stop: stop_dast_ecs_environment + artifacts: + paths: + - environment_url.txt + rules: + - *common_rules + - *ecs_rules + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ + +stop_dast_ecs_environment: + extends: .ecs_image + stage: cleanup + variables: + GIT_STRATEGY: none + script: + - ecs stop-task + allow_failure: true + environment: + name: dast-default + action: stop + needs: + - dast + rules: + - *common_rules + - *ecs_rules + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ + when: always diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index b95b36fd555..a9d9c400a34 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -46,10 +46,10 @@ dependency_scanning: script: - /analyzer run -.cyclone-dx-reports: +.cyclonedx-reports: artifacts: paths: - - "**/cyclonedx-*.json" + - "**/gl-sbom-*.cdx.json" .gemnasium-shared-rule: exists: @@ -66,7 +66,7 @@ dependency_scanning: gemnasium-dependency_scanning: extends: - .ds-analyzer - - .cyclone-dx-reports + - .cyclonedx-reports variables: DS_ANALYZER_NAME: "gemnasium" GEMNASIUM_LIBRARY_SCAN_ENABLED: "true" @@ -81,6 +81,7 @@ gemnasium-dependency_scanning: exists: !reference [.gemnasium-shared-rule, exists] variables: DS_IMAGE_SUFFIX: "-fips" + DS_REMEDIATE: "false" - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdependency_scanning\b/ exists: !reference [.gemnasium-shared-rule, exists] @@ -95,7 +96,7 @@ gemnasium-dependency_scanning: gemnasium-maven-dependency_scanning: extends: - .ds-analyzer - - .cyclone-dx-reports + - .cyclonedx-reports variables: DS_ANALYZER_NAME: "gemnasium-maven" rules: @@ -125,7 +126,7 @@ gemnasium-maven-dependency_scanning: gemnasium-python-dependency_scanning: extends: - .ds-analyzer - - .cyclone-dx-reports + - .cyclonedx-reports variables: DS_ANALYZER_NAME: "gemnasium-python" rules: diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index 9bb2ba69d84..c2d31fd9669 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -20,6 +20,11 @@ .review_ecs_base: stage: review extends: .deploy_to_ecs + after_script: + - echo "http://$(ecs get-task-hostname)" > environment_url.txt + artifacts: + paths: + - environment_url.txt .production_ecs_base: stage: production diff --git a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml index 4687a07d05b..34084272b29 100644 --- a/lib/gitlab/ci/templates/Julia.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Julia.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Julia.gitlab-ci.yml - # This is an example .gitlab-ci.yml file to test (and optionally report the coverage # results of) your [Julia][1] packages. Please refer to the [documentation][2] # for more information about package development in Julia. @@ -12,6 +7,14 @@ # # [1]: http://julialang.org/ # [2]: https://docs.julialang.org/en/v1/manual/documentation/index.html +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Julia.gitlab-ci.yml # Below is the template to run your tests in Julia .test_template: &test_definition diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index 0ec67526234..3a490012f3d 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml index 2f214347ec3..65db649e22f 100644 --- a/lib/gitlab/ci/templates/Mono.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Mono.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Mono.gitlab-ci.yml - # This is a simple gitlab continuous integration template (compatible with the shared runner provided on gitlab.com) # using the official mono docker image to build a visual studio project. # @@ -15,6 +10,14 @@ # # Please find the full example project here: # https://gitlab.com/tobiaskoch/gitlab-ci-example-mono +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Mono.gitlab-ci.yml # see https://hub.docker.com/_/mono/ image: mono:latest diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index 44370f896a7..7a4f7ed628b 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml index 7c8bbe464af..0eb3b483067 100644 --- a/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/OpenShift.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml index 4edc003a638..12640d28d29 100644 --- a/lib/gitlab/ci/templates/PHP.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/PHP.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 690a5a291e1..aab408aa830 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index 390f0bb8061..a83f84da818 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: @@ -26,3 +29,14 @@ test:cargo: script: - rustc --version && cargo --version # Print version info for debugging - cargo test --workspace --verbose + +# Optional: Use a third party library to generate gitlab junit reports +# test:junit-report: +# script: +# Should be specified in Cargo.toml +# - cargo install junitify +# - cargo test -- --format=json -Z unstable-options --report-time | junitify --out $CI_PROJECT_DIR/tests/ +# artifacts: +# when: always +# reports: +# junit: $CI_PROJECT_DIR/tests/*.xml diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml index de54d64dc42..26efe7a8908 100644 --- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml index eedb3b7a310..3c4533d603e 100644 --- a/lib/gitlab/ci/templates/Swift.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Swift.gitlab-ci.yml @@ -1,3 +1,6 @@ +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# # To contribute improvements to CI/CD templates, please follow the Development guide at: # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index 841f17767eb..50ce181095e 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -23,6 +23,9 @@ # You need to have the network drive mapped as Local System user for gitlab-runner service to see it # The best way to persist the mapping is via a scheduled task # running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes +# +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. # place project specific paths in variables to make the rest of the script more generic variables: diff --git a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml index 0b75c298167..58e840da713 100644 --- a/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml @@ -1,8 +1,3 @@ -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml - # This is a very simple template that mainly relies on FastLane to build and distribute your app. # Read more about how to use this template on the blog post https://about.gitlab.com/2019/03/06/ios-publishing-with-gitlab-and-fastlane/ # You will also need fastlane and signing configuration for this to work, along with a MacOS runner. @@ -15,6 +10,14 @@ # https://docs.gitlab.com/runner/security/#usage-of-shell-executor for additional # detail on what to keep in mind in this scenario. +# You can copy and paste this template into a new `.gitlab-ci.yml` file. +# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. + +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/iOS-Fastlane.gitlab-ci.yml + stages: - build - test diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index a452cb197ae..95dff83506d 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -26,7 +26,6 @@ module Gitlab variables.concat(secret_instance_variables) variables.concat(secret_group_variables(environment: environment)) variables.concat(secret_project_variables(environment: environment)) - variables.concat(job.trigger_request.user_variables) if job.trigger_request variables.concat(pipeline.variables) variables.concat(pipeline_schedule_variables) end @@ -52,7 +51,7 @@ module Gitlab # https://gitlab.com/groups/gitlab-org/configure/-/epics/8 # Until then, we need to make both the old and the new KUBECONFIG contexts available collection.concat(deployment_variables(environment: environment, job: job)) - template = ::Ci::GenerateKubeconfigService.new(pipeline, token: job.token).execute + template = ::Ci::GenerateKubeconfigService.new(pipeline, token: job.try(:token)).execute kubeconfig_yaml = collection['KUBECONFIG']&.value template.merge_yaml(kubeconfig_yaml) if kubeconfig_yaml.present? diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 15ebd506055..59acfa80258 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -16,6 +16,14 @@ module Gitlab end def execute + Gitlab::Ci::YamlProcessor::FeatureFlags.with_actor(project) do + parse_config + end + end + + private + + def parse_config if @config_content.blank? return Result.new(errors: ['Please provide content of .gitlab-ci.yml']) end @@ -35,7 +43,9 @@ module Gitlab Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings) end - private + def project + @opts[:project] + end def run_logical_validations! @stages = @ci_config.stages diff --git a/lib/gitlab/ci/yaml_processor/feature_flags.rb b/lib/gitlab/ci/yaml_processor/feature_flags.rb new file mode 100644 index 00000000000..f03db9d0e6b --- /dev/null +++ b/lib/gitlab/ci/yaml_processor/feature_flags.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class YamlProcessor + module FeatureFlags + ACTOR_KEY = 'ci_yaml_processor_feature_flag_actor' + NO_ACTOR_VALUE = :no_actor + + NoActorError = Class.new(StandardError) + NO_ACTOR_MESSAGE = "Actor not set. Ensure to call `enabled?` inside `with_actor` block" + + class << self + # Cache a feature flag actor as thread local variable so + # we can have it available later with #enabled? + def with_actor(actor) + previous = Thread.current[ACTOR_KEY] + + # When actor is `nil` the method `Thread.current[]=` does not + # create the ACTOR_KEY. Instead, we want to still save an explicit + # value to know that we are within the `with_actor` block. + Thread.current[ACTOR_KEY] = actor || NO_ACTOR_VALUE + + yield + ensure + Thread.current[ACTOR_KEY] = previous + end + + # Use this to check if a feature flag is enabled + def enabled?(feature_flag) + ::Feature.enabled?(feature_flag, current_actor) + end + + private + + def current_actor + value = Thread.current[ACTOR_KEY] || (raise NoActorError, NO_ACTOR_MESSAGE) + return if value == NO_ACTOR_VALUE + + value + rescue NoActorError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + + nil + end + end + end + end + end +end diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index e634291f894..5908de68687 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -5,10 +5,10 @@ module Gitlab class PumaWorkerKillerInitializer def self.start( puma_options, - puma_per_worker_max_memory_mb: 1024, - puma_master_max_memory_mb: 800, - additional_puma_dev_max_memory_mb: 200 - ) + puma_per_worker_max_memory_mb: 1200, + puma_master_max_memory_mb: 950, + additional_puma_dev_max_memory_mb: 200) + require 'puma_worker_killer' PumaWorkerKiller.config do |config| diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 574a7dceaa4..8648ffe5f49 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -7,6 +7,8 @@ module Gitlab form_action frame_ancestors frame_src img_src manifest_src media_src object_src report_uri script_src style_src worker_src).freeze + DEFAULT_FALLBACK_VALUE = '<default_value>' + def self.default_enabled Rails.env.development? || Rails.env.test? end @@ -62,8 +64,10 @@ module Gitlab end def initialize(csp_directives) + # Using <default_value> falls back to the default values. + directives = csp_directives.reject { |_, value| value == DEFAULT_FALLBACK_VALUE } @merged_csp_directives = - HashWithIndifferentAccess.new(csp_directives) + HashWithIndifferentAccess.new(directives) .reverse_merge(::Gitlab::ContentSecurityPolicy::ConfigLoader.default_directives) end @@ -134,9 +138,8 @@ module Gitlab def self.allow_sentry(directives) sentry_dsn = Gitlab.config.sentry.clientside_dsn sentry_uri = URI(sentry_dsn) - sentry_uri.user = nil - append_to_directive(directives, 'connect_src', sentry_uri.to_s) + append_to_directive(directives, 'connect_src', "#{sentry_uri.scheme}://#{sentry_uri.host}") end def self.allow_letter_opener(directives) diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index f48ba27888c..4d289a59a6a 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -12,11 +12,8 @@ module Gitlab @contributor = contributor @contributor_time_instance = local_timezone_instance(contributor.timezone).now @current_user = current_user - @projects = if @contributor.include_private_contributions? - ContributedProjectsFinder.new(@contributor).execute(@contributor) - else - ContributedProjectsFinder.new(contributor).execute(current_user) - end + @projects = ContributedProjectsFinder.new(contributor) + .execute(current_user, ignore_visibility: @contributor.include_private_contributions?) end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index 0e6841e10a7..a9c69e3f997 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -5,7 +5,8 @@ module Gitlab module Deployment extend self - def build(deployment, status_changed_at) + # NOTE: Time-sensitive attributes should be explicitly passed as argument instead of reading from database. + def build(deployment, status, status_changed_at) # Deployments will not have a deployable when created using the API. deployable_url = if deployment.deployable @@ -22,9 +23,13 @@ module Gitlab Gitlab::UrlBuilder.build(deployment.deployed_by) end + # `status` argument could be `nil` during the upgrade. We can remove `deployment.status` in GitLab 15.5. + # See https://docs.gitlab.com/ee/development/multi_version_compatibility.html for more info. + deployment_status = status || deployment.status + { object_kind: 'deployment', - status: deployment.status, + status: deployment_status, status_changed_at: status_changed_at, deployment_id: deployment.id, deployable_id: deployment.deployable_id, diff --git a/lib/gitlab/data_builder/issuable.rb b/lib/gitlab/data_builder/issuable.rb index 9a0b964915c..d12537c4874 100644 --- a/lib/gitlab/data_builder/issuable.rb +++ b/lib/gitlab/data_builder/issuable.rb @@ -18,7 +18,7 @@ module Gitlab user: user.hook_attrs, project: issuable.project.hook_attrs, object_attributes: issuable_builder.new(issuable).build, - labels: issuable.labels.map(&:hook_attrs), + labels: issuable.labels_hook_attrs, changes: final_changes(changes.slice(*safe_keys)), # DEPRECATED repository: issuable.project.hook_attrs.slice(:name, :url, :description, :homepage) diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index c13bb1d6a9a..2c124b07006 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -118,6 +118,7 @@ module Gitlab finished_at: build.finished_at, duration: build.duration, queued_duration: build.queued_duration, + failure_reason: (build.failure_reason if build.failed?), when: build.when, manual: build.action?, allow_failure: build.allow_failure, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index b42d164d9c4..8703365b678 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -63,13 +63,45 @@ module Gitlab }.compact.with_indifferent_access.freeze end + # This returns a list of databases that contains all the gitlab_shared schema + # tables. We can't reuse database_base_models because Geo does not support + # the gitlab_shared tables yet. + def self.database_base_models_with_gitlab_shared + @database_base_models_with_gitlab_shared ||= { + # Note that we use ActiveRecord::Base here and not ApplicationRecord. + # This is deliberate, as we also use these classes to apply load + # balancing to, and the load balancer must be enabled for _all_ models + # that inher from ActiveRecord::Base; not just our own models that + # inherit from ApplicationRecord. + main: ::ActiveRecord::Base, + ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil + }.compact.with_indifferent_access.freeze + end + + # This returns a list of databases whose connection supports database load + # balancing. We can't reuse the database_base_models method because the Geo + # database does not support load balancing yet. + # + # TODO: https://gitlab.com/gitlab-org/geo-team/discussions/-/issues/5032 + def self.database_base_models_using_load_balancing + @database_base_models_with_gitlab_shared ||= { + # Note that we use ActiveRecord::Base here and not ApplicationRecord. + # This is deliberate, as we also use these classes to apply load + # balancing to, and the load balancer must be enabled for _all_ models + # that inher from ActiveRecord::Base; not just our own models that + # inherit from ApplicationRecord. + main: ::ActiveRecord::Base, + ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil + }.compact.with_indifferent_access.freeze + end + # This returns a list of base models with connection associated for a given gitlab_schema def self.schemas_to_base_models @schemas_to_base_models ||= { gitlab_main: [self.database_base_models.fetch(:main)], gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main - gitlab_shared: self.database_base_models.values, # all models - gitlab_internal: self.database_base_models.values # all models + gitlab_shared: database_base_models_with_gitlab_shared.values, # all models + gitlab_internal: database_base_models.values # all models }.with_indifferent_access.freeze end @@ -168,7 +200,7 @@ module Gitlab # can potentially upgrade from read to read-write mode (using a different connection), we specify # up-front that we'll explicitly use the primary for the duration of the operation. Gitlab::Database::LoadBalancing::Session.current.use_primary do - base_models = database_base_models.values + base_models = database_base_models_using_load_balancing.values base_models.reduce(block) { |blk, model| -> { model.uncached(&blk) } }.call end end @@ -286,7 +318,7 @@ module Gitlab extend ActiveSupport::Concern class_methods do - # A patch over ActiveRecord::Base.transaction that provides + # A patch over ApplicationRecord.transaction that provides # observability into transactional methods. def transaction(**options, &block) transaction_type = get_transaction_type(connection.transaction_open?, options[:requires_new]) diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index ebc3ee240bd..72aa1cfe00b 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -26,6 +26,7 @@ module Gitlab scope :successful_in_execution_order, -> { where.not(finished_at: nil).with_status(:succeeded).order(:finished_at) } scope :with_preloads, -> { preload(:batched_migration) } scope :created_since, ->(date_time) { where('created_at >= ?', date_time) } + scope :blocked_by_max_attempts, -> { where('attempts >= ?', MAX_ATTEMPTS) } state_machine :status, initial: :pending do state :pending, value: 0 @@ -128,7 +129,8 @@ module Gitlab batched_migration.column_name, batch_min_value: min_value, batch_size: new_batch_size, - job_arguments: batched_migration.job_arguments + job_arguments: batched_migration.job_arguments, + job_class: batched_migration.job_class ) midpoint = next_batch_bounds.last diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index d052d5adc4c..9c8db2243f9 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -104,6 +104,12 @@ module Gitlab .sum(:batch_size) end + def reset_attempts_of_blocked_jobs! + batched_jobs.blocked_by_max_attempts.each_batch(of: 100) do |batch| + batch.update_all(attempts: 0) + end + end + def interval_elapsed?(variance: 0) return true unless last_job @@ -199,10 +205,32 @@ module Gitlab BatchOptimizer.new(self).optimize! end + def health_context + HealthStatus::Context.new([table_name]) + end + def hold!(until_time: 10.minutes.from_now) + duration_s = (until_time - Time.current).round + Gitlab::AppLogger.info( + message: "#{self} put on hold until #{until_time}", + migration_id: id, + job_class_name: job_class_name, + duration_s: duration_s + ) + update!(on_hold_until: until_time) end + def on_hold? + return false unless on_hold_until + + on_hold_until > Time.zone.now + end + + def to_s + "BatchedMigration[id: #{id}]" + end + private def validate_batched_jobs_status diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 388eb596ce2..1bc2e931391 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -29,7 +29,8 @@ module Gitlab if next_batched_job = find_or_create_next_batched_job(active_migration) migration_wrapper.perform(next_batched_job) - active_migration.optimize! + adjust_migration(active_migration) + active_migration.failure! if next_batched_job.failed? && active_migration.should_stop? else finish_active_migration(active_migration) @@ -71,12 +72,17 @@ module Gitlab elsif migration.finished? Gitlab::AppLogger.warn "Batched background migration for the given configuration is already finished: #{configuration}" else + migration.reset_attempts_of_blocked_jobs! + migration.finalize! migration.batched_jobs.with_status(:pending).each { |job| migration_wrapper.perform(job) } run_migration_while(migration, :finalizing) - raise FailedToFinalize unless migration.finished? + error_message = "Batched migration #{migration.job_class_name} could not be completed and a manual action is required."\ + "Check the admin panel at (`/admin/background_migrations`) for more details." + + raise FailedToFinalize, error_message unless migration.finished? end end @@ -101,7 +107,8 @@ module Gitlab active_migration.column_name, batch_min_value: batch_min_value, batch_size: active_migration.batch_size, - job_arguments: active_migration.job_arguments) + job_arguments: active_migration.job_arguments, + job_class: active_migration.job_class) return if next_batch_bounds.nil? @@ -135,6 +142,16 @@ module Gitlab migration.reload_last_job end end + + def adjust_migration(active_migration) + signal = HealthStatus.evaluate(active_migration) + + if signal.is_a?(HealthStatus::Signals::Stop) + active_migration.hold! + else + active_migration.optimize! + end + end end end end diff --git a/lib/gitlab/database/background_migration/health_status.rb b/lib/gitlab/database/background_migration/health_status.rb new file mode 100644 index 00000000000..01f9c5eb5fd --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + # Rather than passing along the migration, we use a more explicitly defined context + Context = Struct.new(:tables) + + def self.evaluate(migration, indicator = Indicators::AutovacuumActiveOnTable) + signal = begin + indicator.new(migration.health_context).evaluate + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e, migration_id: migration.id, + job_class_name: migration.job_class_name) + Signals::Unknown.new(indicator, reason: "unexpected error: #{e.message} (#{e.class})") + end + + log_signal(signal, migration) if signal.log_info? + + signal + end + + def self.log_signal(signal, migration) + Gitlab::AppLogger.info( + message: "#{migration} signaled: #{signal}", + migration_id: migration.id + ) + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/health_status/indicators.rb b/lib/gitlab/database/background_migration/health_status/indicators.rb new file mode 100644 index 00000000000..69503e5b61f --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status/indicators.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + module Indicators + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb b/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb new file mode 100644 index 00000000000..48e12609a13 --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status/indicators/autovacuum_active_on_table.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + module Indicators + class AutovacuumActiveOnTable + def initialize(context) + @context = context + end + + def evaluate + return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? + + autovacuum_active_on = active_autovacuums_for(context.tables) + + if autovacuum_active_on.empty? + Signals::Normal.new(self.class, reason: 'no autovacuum running on any relevant tables') + else + Signals::Stop.new(self.class, reason: "autovacuum running on: #{autovacuum_active_on.join(', ')}") + end + end + + private + + attr_reader :context + + def enabled? + Feature.enabled?(:batched_migrations_health_status_autovacuum, type: :ops) + end + + def active_autovacuums_for(tables) + Gitlab::Database::PostgresAutovacuumActivity.for_tables(tables) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/health_status/signals.rb b/lib/gitlab/database/background_migration/health_status/signals.rb new file mode 100644 index 00000000000..6cd0ebd1bd0 --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status/signals.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + module Signals + # Base class for a signal + class Base + attr_reader :indicator_class, :reason + + def initialize(indicator_class, reason:) + @indicator_class = indicator_class + @reason = reason + end + + def to_s + "#{short_name} (indicator: #{indicator_class}; reason: #{reason})" + end + + # :nocov: + def log_info? + false + end + # :nocov: + + private + + def short_name + self.class.name.demodulize + end + end + + # A Signals::Stop is an indication to put a migration on hold or stop it entirely: + # In general, we want to slow down or pause the migration. + class Stop < Base + # :nocov: + def log_info? + true + end + # :nocov: + end + + # A Signals::Normal indicates normal system state: We carry on with the migration + # and may even attempt to optimize its throughput etc. + class Normal < Base; end + + # When given an Signals::Unknown, something unexpected happened while + # we evaluated system indicators. + class Unknown < Base + # :nocov: + def log_info? + true + end + # :nocov: + end + + # No signal could be determined, e.g. because the indicator + # was disabled. + class NotAvailable < Base; end + end + end + end + end +end diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index 0d876f5124f..02f008abf85 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -36,8 +36,7 @@ module Gitlab private def select_base_models(names) - base_models = Gitlab::Database.database_base_models - + base_models = Gitlab::Database.database_base_models_with_gitlab_shared return base_models if names.empty? names.each_with_object(HashWithIndifferentAccess.new) do |name, hash| @@ -48,7 +47,7 @@ module Gitlab end def with_shared_model_connections(shared_model, selected_databases, &blk) - Gitlab::Database.database_base_models.each_pair do |connection_name, connection_model| + Gitlab::Database.database_base_models_with_gitlab_shared.each_pair do |connection_name, connection_model| if shared_model.limit_connection_names next unless shared_model.limit_connection_names.include?(connection_name.to_sym) end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index baf4cc48424..365a4283d4c 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -13,6 +13,8 @@ module Gitlab module Database module GitlabSchema + GITLAB_SCHEMAS_FILE = 'lib/gitlab/database/gitlab_schemas.yml' + # These tables are deleted/renamed, but still referenced by migrations. # This is needed for now, but should be removed in the future DELETED_TABLES = { @@ -93,7 +95,7 @@ module Gitlab end def self.tables_to_schema - @tables_to_schema ||= YAML.load_file(Rails.root.join('lib/gitlab/database/gitlab_schemas.yml')) + @tables_to_schema ||= YAML.load_file(Rails.root.join(GITLAB_SCHEMAS_FILE)) end def self.schema_names @@ -102,3 +104,5 @@ module Gitlab end end end + +Gitlab::Database::GitlabSchema.prepend_mod diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 71c323cb393..4a467d18f0a 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -108,10 +108,12 @@ ci_resource_groups: :gitlab_ci ci_resources: :gitlab_ci ci_runner_namespaces: :gitlab_ci ci_runner_projects: :gitlab_ci +ci_runner_versions: :gitlab_ci ci_runners: :gitlab_ci ci_running_builds: :gitlab_ci ci_sources_pipelines: :gitlab_ci ci_secure_files: :gitlab_ci +ci_secure_file_states: :gitlab_ci ci_sources_projects: :gitlab_ci ci_stages: :gitlab_ci ci_subscriptions_projects: :gitlab_ci @@ -131,7 +133,6 @@ cluster_providers_gcp: :gitlab_main clusters_applications_cert_managers: :gitlab_main clusters_applications_cilium: :gitlab_main clusters_applications_crossplane: :gitlab_main -clusters_applications_elastic_stacks: :gitlab_main clusters_applications_helm: :gitlab_main clusters_applications_ingress: :gitlab_main clusters_applications_jupyter: :gitlab_main @@ -139,7 +140,6 @@ clusters_applications_knative: :gitlab_main clusters_applications_prometheus: :gitlab_main clusters_applications_runners: :gitlab_main clusters: :gitlab_main -clusters_integration_elasticstack: :gitlab_main clusters_integration_prometheus: :gitlab_main clusters_kubernetes_namespaces: :gitlab_main commit_user_mentions: :gitlab_main @@ -325,6 +325,7 @@ milestone_releases: :gitlab_main milestones: :gitlab_main namespace_admin_notes: :gitlab_main namespace_aggregation_schedules: :gitlab_main +namespace_bans: :gitlab_main namespace_limits: :gitlab_main namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main @@ -423,6 +424,8 @@ project_incident_management_settings: :gitlab_main project_metrics_settings: :gitlab_main project_mirror_data: :gitlab_main project_pages_metadata: :gitlab_main +project_relation_export_uploads: :gitlab_main +project_relation_exports: :gitlab_main project_repositories: :gitlab_main project_repository_states: :gitlab_main project_repository_storage_moves: :gitlab_main @@ -432,7 +435,6 @@ projects: :gitlab_main projects_sync_events: :gitlab_main project_statistics: :gitlab_main project_topics: :gitlab_main -project_tracing_settings: :gitlab_main prometheus_alert_events: :gitlab_main prometheus_alerts: :gitlab_main prometheus_metrics: :gitlab_main @@ -467,6 +469,10 @@ routes: :gitlab_main saml_group_links: :gitlab_main saml_providers: :gitlab_main saved_replies: :gitlab_main +sbom_components: :gitlab_main +sbom_occurrences: :gitlab_main +sbom_component_versions: :gitlab_main +sbom_sources: :gitlab_main schema_migrations: :gitlab_internal scim_identities: :gitlab_main scim_oauth_access_tokens: :gitlab_main @@ -547,6 +553,7 @@ vulnerability_flags: :gitlab_main vulnerability_historical_statistics: :gitlab_main vulnerability_identifiers: :gitlab_main vulnerability_issue_links: :gitlab_main +vulnerability_merge_request_links: :gitlab_main vulnerability_occurrence_identifiers: :gitlab_main vulnerability_occurrence_pipelines: :gitlab_main vulnerability_occurrences: :gitlab_main @@ -571,3 +578,4 @@ zentao_tracker_data: :gitlab_main dingtalk_tracker_data: :gitlab_main zoom_meetings: :gitlab_main batched_background_migration_job_transition_logs: :gitlab_shared +user_namespace_callouts: :gitlab_main diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index 6517923d23e..5f9416fb4db 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -19,7 +19,7 @@ module Gitlab ].freeze def self.base_models - @base_models ||= ::Gitlab::Database.database_base_models.values.freeze + @base_models ||= ::Gitlab::Database.database_base_models_using_load_balancing.values.freeze end def self.each_load_balancer diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 4aaeaa7b365..936b986ea07 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -72,12 +72,6 @@ module Gitlab ) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! - if transaction_open? - raise 'The `queue_batched_background_migration` cannot be run inside a transaction. ' \ - 'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \ - 'your migration class.' - end - gitlab_schema ||= gitlab_schema_from_context Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information @@ -182,12 +176,6 @@ module Gitlab def delete_batched_background_migration(job_class_name, table_name, column_name, job_arguments) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! - if transaction_open? - raise 'The `#delete_batched_background_migration` cannot be run inside a transaction. ' \ - 'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \ - 'your migration class.' - end - Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information Gitlab::Database::BackgroundMigration::BatchedMigration diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index 0c6a8d3d856..f38d847b0e8 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -14,7 +14,7 @@ module Gitlab def jobs_by_migration_name Gitlab::Database::BackgroundMigration::BatchedMigration .executable - .created_after(2.days.ago) # Simple way to exclude migrations already running before migration testing + .created_after(3.hours.ago) # Simple way to exclude migrations already running before migration testing .to_h do |migration| batching_strategy = migration.batch_class.new(connection: connection) diff --git a/lib/gitlab/database/postgres_autovacuum_activity.rb b/lib/gitlab/database/postgres_autovacuum_activity.rb new file mode 100644 index 00000000000..a4dc199c259 --- /dev/null +++ b/lib/gitlab/database/postgres_autovacuum_activity.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PostgresAutovacuumActivity < SharedModel + self.table_name = 'postgres_autovacuum_activity' + self.primary_key = 'table_identifier' + + def self.for_tables(tables) + Gitlab::Database::LoadBalancing::Session.current.use_primary do + where('schema = current_schema()').where(table: tables) + end + end + + def to_s + "table #{table_identifier} (started: #{vacuum_start})" + end + end + end +end diff --git a/lib/gitlab/database_importers/instance_administrators/create_group.rb b/lib/gitlab/database_importers/instance_administrators/create_group.rb index 79244120776..bb489ced3d2 100644 --- a/lib/gitlab/database_importers/instance_administrators/create_group.rb +++ b/lib/gitlab/database_importers/instance_administrators/create_group.rb @@ -77,7 +77,7 @@ module Gitlab def add_group_members(result) group = result[:group] - members = group.add_users(members_to_add(group), Gitlab::Access::MAINTAINER) + members = group.add_members(members_to_add(group), Gitlab::Access::MAINTAINER) errors = members.flat_map { |member| member.errors.full_messages } if errors.any? @@ -112,7 +112,7 @@ module Gitlab def members_to_add(group) # Exclude admins who are already members of group because - # `group.add_users(users)` returns an error if the users parameter contains + # `group.add_members(users)` returns an error if the users parameter contains # users who are already members of the group. instance_admins - group.members.collect(&:user) end diff --git a/lib/gitlab/dependency_linker/base_linker.rb b/lib/gitlab/dependency_linker/base_linker.rb index 36a840372c5..76855f2950d 100644 --- a/lib/gitlab/dependency_linker/base_linker.rb +++ b/lib/gitlab/dependency_linker/base_linker.rb @@ -6,9 +6,6 @@ module Gitlab URL_REGEX = %r{https?://[^'" ]+}.freeze GIT_INVALID_URL_REGEX = /^git\+#{URL_REGEX}/.freeze REPO_REGEX = %r{[^/'" ]+/[^/'" ]+}.freeze - VALID_LINK_ATTRIBUTES = %w[href rel target].freeze - - include ActionView::Helpers::SanitizeHelper class_attribute :file_type @@ -65,10 +62,9 @@ module Gitlab end def link_tag(name, url) - sanitize( - %{<a href="#{ERB::Util.html_escape_once(url)}" rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}, - attributes: VALID_LINK_ATTRIBUTES - ) + href_attribute = %{href="#{ERB::Util.html_escape_once(url)}" } if Gitlab::UrlSanitizer.valid_web?(url) + + %{<a #{href_attribute}rel="nofollow noreferrer noopener" target="_blank">#{ERB::Util.html_escape_once(name)}</a>}.html_safe end # Links package names based on regex. diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 8e039d32ef5..8c55652da43 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -373,7 +373,7 @@ module Gitlab end def rendered - return unless use_semantic_ipynb_diff? && ipynb? && modified_file? && !too_large? + return unless use_semantic_ipynb_diff? && ipynb? && modified_file? && !collapsed? && !too_large? strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) } end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 7fa1bd6b5ec..924de132840 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -68,20 +68,12 @@ module Gitlab end end - def diff_file_with_old_path(old_path, a_mode = nil) - if Feature.enabled?(:file_identifier_hash) && a_mode.present? - diff_files.find { |diff_file| diff_file.old_path == old_path && diff_file.a_mode == a_mode } - else - diff_files.find { |diff_file| diff_file.old_path == old_path } - end + def diff_file_with_old_path(old_path) + diff_files.find { |diff_file| diff_file.old_path == old_path } end - def diff_file_with_new_path(new_path, b_mode = nil) - if Feature.enabled?(:file_identifier_hash) && b_mode.present? - diff_files.find { |diff_file| diff_file.new_path == new_path && diff_file.b_mode == b_mode } - else - diff_files.find { |diff_file| diff_file.new_path == new_path } - end + def diff_file_with_new_path(new_path) + diff_files.find { |diff_file| diff_file.new_path == new_path } end def clear_cache diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb index e24150a2330..19fc028594c 100644 --- a/lib/gitlab/diff/formatters/base_formatter.rb +++ b/lib/gitlab/diff/formatters/base_formatter.rb @@ -6,7 +6,6 @@ module Gitlab class BaseFormatter attr_reader :old_path attr_reader :new_path - attr_reader :file_identifier_hash attr_reader :base_sha attr_reader :start_sha attr_reader :head_sha @@ -16,7 +15,6 @@ module Gitlab attrs[:diff_refs] = diff_file.diff_refs attrs[:old_path] = diff_file.old_path attrs[:new_path] = diff_file.new_path - attrs[:file_identifier_hash] = diff_file.file_identifier_hash end if diff_refs = attrs[:diff_refs] @@ -27,7 +25,6 @@ module Gitlab @old_path = attrs[:old_path] @new_path = attrs[:new_path] - @file_identifier_hash = attrs[:file_identifier_hash] @base_sha = attrs[:base_sha] @start_sha = attrs[:start_sha] @head_sha = attrs[:head_sha] @@ -38,7 +35,7 @@ module Gitlab end def to_h - out = { + { base_sha: base_sha, start_sha: start_sha, head_sha: head_sha, @@ -46,12 +43,6 @@ module Gitlab new_path: new_path, position_type: position_type } - - if Feature.enabled?(:file_identifier_hash) - out[:file_identifier_hash] = file_identifier_hash - end - - out end def position_type diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index f950d01fdf0..8e9dc3a305f 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -193,6 +193,8 @@ module Gitlab results = redis.hmget(key, file_paths) end + record_hit_ratio(results) + results.map! do |result| Gitlab::Json.parse(gzip_decompress(result), symbolize_names: true) unless result.nil? end @@ -215,6 +217,11 @@ module Gitlab def current_transaction ::Gitlab::Metrics::WebTransaction.current end + + def record_hit_ratio(results) + current_transaction&.increment(:gitlab_redis_diff_caching_requests_total) + current_transaction&.increment(:gitlab_redis_diff_caching_hits_total) if results.any?(&:present?) + end end end end diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 74c33c46598..40b6ae2f14e 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -9,7 +9,6 @@ module Gitlab delegate :old_path, :new_path, - :file_identifier_hash, :base_sha, :start_sha, :head_sha, @@ -161,11 +160,7 @@ module Gitlab def find_diff_file_from(diffable) diff_files = diffable.diffs(diff_options).diff_files - if Feature.enabled?(:file_identifier_hash) && file_identifier_hash.present? - diff_files.find { |df| df.file_identifier_hash == file_identifier_hash } - else - diff_files.first - end + diff_files.first end def multiline? diff --git a/lib/gitlab/diff/position_tracer/image_strategy.rb b/lib/gitlab/diff/position_tracer/image_strategy.rb index 046a6782dda..aac52b536f7 100644 --- a/lib/gitlab/diff/position_tracer/image_strategy.rb +++ b/lib/gitlab/diff/position_tracer/image_strategy.rb @@ -7,24 +7,21 @@ module Gitlab def trace(position) a_path = position.old_path b_path = position.new_path - diff_file = diff_file(position) - a_mode = diff_file&.a_mode - b_mode = diff_file&.b_mode # If file exists in B->D (e.g. updated, renamed, removed), let the # note become outdated. - bd_diff = bd_diffs.diff_file_with_old_path(b_path, b_mode) + bd_diff = bd_diffs.diff_file_with_old_path(b_path) return { position: new_position(position, bd_diff), outdated: true } if bd_diff # If file still exists in the new diff, update the position. - cd_diff = cd_diffs.diff_file_with_new_path(b_path, b_mode) + cd_diff = cd_diffs.diff_file_with_new_path(b_path) return { position: new_position(position, cd_diff), outdated: false } if cd_diff # If file exists in A->C (e.g. rebased and same changes were present # in target branch), let the note become outdated. - ac_diff = ac_diffs.diff_file_with_old_path(a_path, a_mode) + ac_diff = ac_diffs.diff_file_with_old_path(a_path) return { position: new_position(position, ac_diff), outdated: true } if ac_diff diff --git a/lib/gitlab/diff/position_tracer/line_strategy.rb b/lib/gitlab/diff/position_tracer/line_strategy.rb index 0f0b8f0c4f3..d7a7e3f5425 100644 --- a/lib/gitlab/diff/position_tracer/line_strategy.rb +++ b/lib/gitlab/diff/position_tracer/line_strategy.rb @@ -76,20 +76,16 @@ module Gitlab def trace_added_line(position) b_path = position.new_path b_line = position.new_line - diff_file = diff_file(position) - b_mode = diff_file&.b_mode - bd_diff = bd_diffs.diff_file_with_old_path(b_path, b_mode) + bd_diff = bd_diffs.diff_file_with_old_path(b_path) d_path = bd_diff&.new_path || b_path - d_mode = bd_diff&.b_mode || b_mode d_line = LineMapper.new(bd_diff).old_to_new(b_line) if d_line - cd_diff = cd_diffs.diff_file_with_new_path(d_path, d_mode) + cd_diff = cd_diffs.diff_file_with_new_path(d_path) c_path = cd_diff&.old_path || d_path - c_mode = cd_diff&.a_mode || d_mode c_line = LineMapper.new(cd_diff).new_to_old(d_line) if c_line @@ -102,7 +98,7 @@ module Gitlab else # If the line is no longer in the MR, we unfortunately cannot show # the current state on the CD diff, so we treat it as outdated. - ac_diff = ac_diffs.diff_file_with_new_path(c_path, c_mode) + ac_diff = ac_diffs.diff_file_with_new_path(c_path) { position: new_position(ac_diff, nil, c_line, position.line_range), outdated: true } end @@ -119,26 +115,22 @@ module Gitlab def trace_removed_line(position) a_path = position.old_path a_line = position.old_line - diff_file = diff_file(position) - a_mode = diff_file&.a_mode - ac_diff = ac_diffs.diff_file_with_old_path(a_path, a_mode) + ac_diff = ac_diffs.diff_file_with_old_path(a_path) c_path = ac_diff&.new_path || a_path - c_mode = ac_diff&.b_mode || a_mode c_line = LineMapper.new(ac_diff).old_to_new(a_line) if c_line - cd_diff = cd_diffs.diff_file_with_old_path(c_path, c_mode) + cd_diff = cd_diffs.diff_file_with_old_path(c_path) d_path = cd_diff&.new_path || c_path - d_mode = cd_diff&.b_mode || c_mode d_line = LineMapper.new(cd_diff).old_to_new(c_line) if d_line # If the line is still in C but also in D, it has turned from a # removed line into an unchanged one. - bd_diff = bd_diffs.diff_file_with_new_path(d_path, d_mode) + bd_diff = bd_diffs.diff_file_with_new_path(d_path) { position: new_position(bd_diff, nil, d_line, position.line_range), outdated: true } else @@ -156,21 +148,17 @@ module Gitlab a_line = position.old_line b_path = position.new_path b_line = position.new_line - diff_file = diff_file(position) - a_mode = diff_file&.a_mode - b_mode = diff_file&.b_mode - ac_diff = ac_diffs.diff_file_with_old_path(a_path, a_mode) + ac_diff = ac_diffs.diff_file_with_old_path(a_path) c_path = ac_diff&.new_path || a_path - c_mode = ac_diff&.b_mode || a_mode c_line = LineMapper.new(ac_diff).old_to_new(a_line) - bd_diff = bd_diffs.diff_file_with_old_path(b_path, b_mode) + bd_diff = bd_diffs.diff_file_with_old_path(b_path) d_line = LineMapper.new(bd_diff).old_to_new(b_line) - cd_diff = cd_diffs.diff_file_with_old_path(c_path, c_mode) + cd_diff = cd_diffs.diff_file_with_old_path(c_path) if c_line && d_line # If the line is still in C and D, it is still unchanged. diff --git a/lib/gitlab/diff/rendered/notebook/diff_file.rb b/lib/gitlab/diff/rendered/notebook/diff_file.rb index 0a5b2ec3890..3e1652bd318 100644 --- a/lib/gitlab/diff/rendered/notebook/diff_file.rb +++ b/lib/gitlab/diff/rendered/notebook/diff_file.rb @@ -79,7 +79,7 @@ module Gitlab rescue Timeout::Error => e rendered_timeout.increment(source: Gitlab::Runtime.sidekiq? ? BACKGROUND_EXECUTION : FOREGROUND_EXECUTION) log_event(LOG_IPYNBDIFF_TIMEOUT, e) - rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e + rescue IpynbDiff::InvalidNotebookError => e log_event(LOG_IPYNBDIFF_INVALID, e) end end diff --git a/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb b/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb index 2e1b5ea301d..f381792953e 100644 --- a/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb +++ b/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb @@ -91,7 +91,7 @@ module Gitlab return 0 unless line_in_source.present? - line_in_source + 1 + line_in_source end def image_as_rich_text(line_text) diff --git a/lib/gitlab/elasticsearch/logs/lines.rb b/lib/gitlab/elasticsearch/logs/lines.rb deleted file mode 100644 index ff9185dd331..00000000000 --- a/lib/gitlab/elasticsearch/logs/lines.rb +++ /dev/null @@ -1,157 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Elasticsearch - module Logs - class Lines - InvalidCursor = Class.new(RuntimeError) - - # How many log lines to fetch in a query - LOGS_LIMIT = 500 - - def initialize(client) - @client = client - end - - def pod_logs(namespace, pod_name: nil, container_name: nil, search: nil, start_time: nil, end_time: nil, cursor: nil, chart_above_v2: true) - query = { bool: { must: [] } }.tap do |q| - filter_pod_name(q, pod_name) - filter_namespace(q, namespace) - filter_container_name(q, container_name) - filter_search(q, search) - filter_times(q, start_time, end_time) - end - - body = build_body(query, cursor, chart_above_v2) - response = @client.search body: body - - format_response(response) - end - - private - - def build_body(query, cursor = nil, chart_above_v2 = true) - offset_field = chart_above_v2 ? "log.offset" : "offset" - body = { - query: query, - # reverse order so we can query N-most recent records - sort: [ - { "@timestamp": { order: :desc } }, - { "#{offset_field}": { order: :desc } } - ], - # only return these fields in the response - _source: ["@timestamp", "message", "kubernetes.pod.name"], - # fixed limit for now, we should support paginated queries - size: ::Gitlab::Elasticsearch::Logs::Lines::LOGS_LIMIT - } - - unless cursor.nil? - body[:search_after] = decode_cursor(cursor) - end - - body - end - - def filter_pod_name(query, pod_name) - # We can filter by "all pods" with a null pod_name - return if pod_name.nil? - - query[:bool][:must] << { - match_phrase: { - "kubernetes.pod.name" => { - query: pod_name - } - } - } - end - - def filter_namespace(query, namespace) - query[:bool][:must] << { - match_phrase: { - "kubernetes.namespace" => { - query: namespace - } - } - } - end - - def filter_container_name(query, container_name) - # A pod can contain multiple containers. - # By default we return logs from every container - return if container_name.nil? - - query[:bool][:must] << { - match_phrase: { - "kubernetes.container.name" => { - query: container_name - } - } - } - end - - def filter_search(query, search) - return if search.nil? - - query[:bool][:must] << { - simple_query_string: { - query: search, - fields: [:message], - default_operator: :and - } - } - end - - def filter_times(query, start_time, end_time) - return unless start_time || end_time - - time_range = { range: { :@timestamp => {} } }.tap do |tr| - tr[:range][:@timestamp][:gte] = start_time if start_time - tr[:range][:@timestamp][:lt] = end_time if end_time - end - - query[:bool][:filter] = [time_range] - end - - def format_response(response) - results = response.fetch("hits", {}).fetch("hits", []) - last_result = results.last - results = results.map do |hit| - { - timestamp: hit["_source"]["@timestamp"], - message: hit["_source"]["message"], - pod: hit["_source"]["kubernetes"]["pod"]["name"] - } - end - - # we queried for the N-most recent records but we want them ordered oldest to newest - { - logs: results.reverse, - cursor: last_result.nil? ? nil : encode_cursor(last_result["sort"]) - } - end - - # we want to hide the implementation details of the search_after parameter from the frontend - # behind a single easily transmitted value - def encode_cursor(obj) - obj.join(',') - end - - def decode_cursor(obj) - cursor = obj.split(',').map(&:to_i) - - unless valid_cursor(cursor) - raise InvalidCursor, "invalid cursor format" - end - - cursor - end - - def valid_cursor(cursor) - cursor.instance_of?(Array) && - cursor.length == 2 && - cursor.map {|i| i.instance_of?(Integer)}.reduce(:&) - end - end - end - end -end diff --git a/lib/gitlab/elasticsearch/logs/pods.rb b/lib/gitlab/elasticsearch/logs/pods.rb deleted file mode 100644 index 66499ae956a..00000000000 --- a/lib/gitlab/elasticsearch/logs/pods.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Elasticsearch - module Logs - class Pods - # How many items to fetch in a query - PODS_LIMIT = 500 - CONTAINERS_LIMIT = 500 - - def initialize(client) - @client = client - end - - def pods(namespace) - body = build_body(namespace) - response = @client.search body: body - - format_response(response) - end - - private - - def build_body(namespace) - { - aggs: { - pods: { - aggs: { - containers: { - terms: { - field: 'kubernetes.container.name', - size: ::Gitlab::Elasticsearch::Logs::Pods::CONTAINERS_LIMIT - } - } - }, - terms: { - field: 'kubernetes.pod.name', - size: ::Gitlab::Elasticsearch::Logs::Pods::PODS_LIMIT - } - } - }, - query: { - bool: { - must: { - match_phrase: { - "kubernetes.namespace": namespace - } - } - } - }, - # don't populate hits, only the aggregation is needed - size: 0 - } - end - - def format_response(response) - results = response.dig("aggregations", "pods", "buckets") || [] - results.map do |bucket| - { - name: bucket["key"], - container_names: (bucket.dig("containers", "buckets") || []).map do |cbucket| - cbucket["key"] - end - } - end - end - end - end - end -end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 71b1d4ed8f9..8e2c7559bc1 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -98,7 +98,10 @@ module Gitlab title: mail.subject, description: message_including_template, confidential: true, - external_author: from_address + external_author: from_address, + extra_params: { + cc: mail.cc + } }, spam_params: nil ).execute diff --git a/lib/gitlab/email/message/in_product_marketing/experience.rb b/lib/gitlab/email/message/in_product_marketing/experience.rb deleted file mode 100644 index 7520de6d2a3..00000000000 --- a/lib/gitlab/email/message/in_product_marketing/experience.rb +++ /dev/null @@ -1,92 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Message - module InProductMarketing - class Experience < Base - include Gitlab::Utils::StrongMemoize - - EASE_SCORE_SURVEY_ID = 1 - - def subject_line - s_('InProductMarketing|Do you have a minute?') - end - - def tagline - end - - def title - s_('InProductMarketing|We want your GitLab experience to be great') - end - - def subtitle - s_('InProductMarketing|Take this 1-question survey!') - end - - def body_line1 - s_('InProductMarketing|%{strong_start}Overall, how difficult or easy was it to get started with GitLab?%{strong_end}').html_safe % strong_options - end - - def body_line2 - s_('InProductMarketing|Click on the number below that corresponds with your answer — 1 being very difficult, 5 being very easy.') - end - - def cta_text - end - - def feedback_link(rating) - params = { - onboarding_progress: onboarding_progress, - response: rating, - show_invite_link: show_invite_link, - survey_id: EASE_SCORE_SURVEY_ID - } - - params[:show_incentive] = true if show_incentive? - - "#{gitlab_com_root_url}/-/survey_responses?#{params.to_query}" - end - - def feedback_ratings(rating) - [ - s_('InProductMarketing|Very difficult'), - s_('InProductMarketing|Difficult'), - s_('InProductMarketing|Neutral'), - s_('InProductMarketing|Easy'), - s_('InProductMarketing|Very easy') - ][rating - 1] - end - - def feedback_thanks - s_('InProductMarketing|Feedback from users like you really improves our product. Thanks for your help!') - end - - private - - def onboarding_progress - strong_memoize(:onboarding_progress) do - group.onboarding_progress.number_of_completed_actions - end - end - - def show_invite_link - strong_memoize(:show_invite_link) do - group.max_member_access_for_user(user) >= GroupMember::DEVELOPER && user.preferred_language == 'en' - end - end - - def show_incentive? - show_invite_link && group.member_count > 1 - end - - def gitlab_com_root_url - return root_url.chomp('/') if Rails.env.development? - - Gitlab::Saas.com_url - end - end - end - end - end -end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 35c1a1e73cf..83920182da4 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -20,7 +20,10 @@ module Gitlab ::Gitlab::ErrorTracking::Processor::SidekiqProcessor, ::Gitlab::ErrorTracking::Processor::GrpcErrorProcessor, ::Gitlab::ErrorTracking::Processor::ContextPayloadProcessor, - ::Gitlab::ErrorTracking::Processor::SanitizeErrorMessageProcessor + ::Gitlab::ErrorTracking::Processor::SanitizeErrorMessageProcessor, + # IMPORTANT: this processor must stay at the bottom, right before + # sending the event to Sentry. + ::Gitlab::ErrorTracking::Processor::SanitizerProcessor ].freeze class << self diff --git a/lib/gitlab/error_tracking/error_repository.rb b/lib/gitlab/error_tracking/error_repository.rb index 4ec636703d9..fd2467add20 100644 --- a/lib/gitlab/error_tracking/error_repository.rb +++ b/lib/gitlab/error_tracking/error_repository.rb @@ -15,7 +15,12 @@ module Gitlab # # @return [self] def self.build(project) - strategy = ActiveRecordStrategy.new(project) + strategy = + if Feature.enabled?(:use_click_house_database_for_error_tracking, project) + OpenApiStrategy.new(project) + else + ActiveRecordStrategy.new(project) + end new(strategy) end @@ -72,14 +77,15 @@ module Gitlab # @param sort [String] order list by 'first_seen', 'last_seen', or 'frequency' # @param filters [Hash<Symbol, String>] filter list by # @option filters [String] :status error status + # @params query [String, nil] free text search # @param limit [Integer, String] limit result # @param cursor [Hash] pagination information # # @return [Array<Array<Gitlab::ErrorTracking::Error>, Pagination>] - def list_errors(sort: 'last_seen', filters: {}, limit: 20, cursor: {}) + def list_errors(sort: 'last_seen', filters: {}, query: nil, limit: 20, cursor: {}) limit = [limit.to_i, 100].min - strategy.list_errors(filters: filters, sort: sort, limit: limit, cursor: cursor) + strategy.list_errors(filters: filters, query: query, sort: sort, limit: limit, cursor: cursor) end # Fetches last event for error +id+. @@ -105,6 +111,10 @@ module Gitlab strategy.update_error(id, status: status) end + def dsn_url(public_key) + strategy.dsn_url(public_key) + end + private attr_reader :strategy diff --git a/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb b/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb index e5b532ee0f0..01e7fbda384 100644 --- a/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb +++ b/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb @@ -39,11 +39,12 @@ module Gitlab handle_exceptions(e) end - def list_errors(filters:, sort:, limit:, cursor:) + def list_errors(filters:, query:, sort:, limit:, cursor:) errors = project_errors errors = filter_by_status(errors, filters[:status]) errors = sort(errors, sort) errors = errors.keyset_paginate(cursor: cursor, per_page: limit) + # query is not supported pagination = ErrorRepository::Pagination.new(errors.cursor_for_next_page, errors.cursor_for_previous_page) @@ -60,6 +61,24 @@ module Gitlab project_error(id).update(attributes) end + def dsn_url(public_key) + gitlab = Settings.gitlab + + custom_port = Settings.gitlab_on_standard_port? ? nil : ":#{gitlab.port}" + + base_url = [ + gitlab.protocol, + "://", + public_key, + '@', + gitlab.host, + custom_port, + gitlab.relative_url_root + ].join('') + + "#{base_url}/api/v4/error_tracking/collector/#{project.id}" + end + private attr_reader :project diff --git a/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb new file mode 100644 index 00000000000..e3eae20c520 --- /dev/null +++ b/lib/gitlab/error_tracking/error_repository/open_api_strategy.rb @@ -0,0 +1,248 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + class ErrorRepository + class OpenApiStrategy + def initialize(project) + @project = project + + api_url = configured_api_url + + open_api.configure do |config| + config.scheme = api_url.scheme + config.host = [api_url.host, api_url.port].compact.join(':') + config.server_index = nil + config.logger = Gitlab::AppLogger + end + end + + def report_error( + name:, description:, actor:, platform:, + environment:, level:, occurred_at:, payload: + ) + raise NotImplementedError, 'Use ingestion endpoint' + end + + def find_error(id) + api = open_api::ErrorsApi.new + error = api.get_error(project_id, id) + + to_sentry_detailed_error(error) + rescue ErrorTrackingOpenAPI::ApiError => e + log_exception(e) + nil + end + + def list_errors(filters:, query:, sort:, limit:, cursor:) + opts = { + sort: "#{sort}_desc", + status: filters[:status], + query: query, + cursor: cursor, + limit: limit + }.compact + + api = open_api::ErrorsApi.new + errors, _status, headers = api.list_errors_with_http_info(project_id, opts) + pagination = pagination_from_headers(headers) + + if errors.size < limit + # Don't show next link if amount of errors is less then requested. + # This a workaround until the Golang backend returns link cursor + # only if there is a next page. + pagination.next = nil + end + + [errors.map { to_sentry_error(_1) }, pagination] + rescue ErrorTrackingOpenAPI::ApiError => e + log_exception(e) + [[], ErrorRepository::Pagination.new] + end + + def last_event_for(id) + event = newest_event_for(id) + return unless event + + api = open_api::ErrorsApi.new + error = api.get_error(project_id, id) + return unless error + + to_sentry_error_event(event, error) + rescue ErrorTrackingOpenAPI::ApiError => e + log_exception(e) + nil + end + + def update_error(id, **attributes) + opts = attributes.slice(:status) + + body = open_api::ErrorUpdatePayload.new(opts) + + api = open_api::ErrorsApi.new + api.update_error(project_id, id, body) + + true + rescue ErrorTrackingOpenAPI::ApiError => e + log_exception(e) + false + end + + def dsn_url(public_key) + config = open_api::Configuration.default + + base_url = [ + config.scheme, + "://", + public_key, + '@', + config.host, + config.base_path + ].join('') + + "#{base_url}/projects/api/#{project_id}" + end + + private + + def event_for(id, sort:) + opts = { sort: sort, limit: 1 } + + api = open_api::ErrorsApi.new + api.list_events(project_id, id, opts).first + rescue ErrorTrackingOpenAPI::ApiError => e + log_exception(e) + nil + end + + def newest_event_for(id) + event_for(id, sort: 'occurred_at_desc') + end + + def oldest_event_for(id) + event_for(id, sort: 'occurred_at_asc') + end + + def to_sentry_error(error) + Gitlab::ErrorTracking::Error.new( + id: error.fingerprint.to_s, + title: error.name, + message: error.description, + culprit: error.actor, + first_seen: error.first_seen_at, + last_seen: error.last_seen_at, + status: error.status, + count: error.event_count, + user_count: error.approximated_user_count + ) + end + + def to_sentry_detailed_error(error) + Gitlab::ErrorTracking::DetailedError.new( + id: error.fingerprint.to_s, + title: error.name, + message: error.description, + culprit: error.actor, + first_seen: error.first_seen_at.to_s, + last_seen: error.last_seen_at.to_s, + count: error.event_count, + user_count: error.approximated_user_count, + project_id: error.project_id, + status: error.status, + tags: { level: nil, logger: nil }, + external_url: external_url(error.fingerprint), + external_base_url: external_base_url, + integrated: true, + first_release_version: release_from(oldest_event_for(error.fingerprint)), + last_release_version: release_from(newest_event_for(error.fingerprint)) + ) + end + + def to_sentry_error_event(event, error) + Gitlab::ErrorTracking::ErrorEvent.new( + issue_id: event.fingerprint.to_s, + date_received: error.last_seen_at, + stack_trace_entries: build_stacktrace(event) + ) + end + + def pagination_from_headers(headers) + links = headers['link'].to_s.split(', ') + + pagination_hash = links.map { parse_pagination_link(_1) }.compact.to_h + + ErrorRepository::Pagination.new(pagination_hash['next'], pagination_hash['prev']) + end + + LINK_PATTERN = %r{cursor=(?<cursor>[^&]+).*; rel="(?<direction>\w+)"}.freeze + + def parse_pagination_link(content) + match = LINK_PATTERN.match(content) + return unless match + + [match['direction'], CGI.unescape(match['cursor'])] + end + + def build_stacktrace(event) + payload = parse_json(event.payload) + return [] unless payload + + ::ErrorTracking::StacktraceBuilder.new(payload).stacktrace + end + + def parse_json(payload) + Gitlab::Json.parse(payload) + rescue JSON::ParserError + end + + def release_from(event) + return unless event + + payload = parse_json(event.payload) + return unless payload + + payload['release'] + end + + def project_id + @project.id + end + + def open_api + ErrorTrackingOpenAPI + end + + # For compatibility with sentry integration + def external_url(id) + Gitlab::Routing.url_helpers.details_namespace_project_error_tracking_index_url( + namespace_id: @project.namespace, + project_id: @project, + issue_id: id) + end + + # For compatibility with sentry integration + def external_base_url + Gitlab::Routing.url_helpers.project_url(@project) + end + + def configured_api_url + url = Gitlab::CurrentSettings.current_application_settings.error_tracking_api_url || + 'http://localhost:8080' + + Gitlab::UrlBlocker.validate!(url, schemes: %w[http https], allow_localhost: true) + + URI(url) + end + + def log_exception(exception) + params = { + http_code: exception.code, + response_body: exception.response_body&.truncate(100) + } + + Gitlab::AppLogger.error(Gitlab::Utils::InlineHash.merge_keys(params, prefix: 'open_api')) + end + end + end + end +end diff --git a/lib/gitlab/error_tracking/processor/sanitizer_processor.rb b/lib/gitlab/error_tracking/processor/sanitizer_processor.rb new file mode 100644 index 00000000000..e6114f8e206 --- /dev/null +++ b/lib/gitlab/error_tracking/processor/sanitizer_processor.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module ErrorTracking + module Processor + module SanitizerProcessor + SANITIZED_HTTP_HEADERS = %w[Authorization Private-Token Job-Token].freeze + SANITIZED_ATTRIBUTES = %i[user contexts extra tags].freeze + + # This processor removes sensitive fields or headers from the event + # before sending. Sentry versions above 4.0 don't support + # sanitized_fields and sanitized_http_headers anymore. The official + # document recommends using before_send instead. + # + # For more information, please visit: + # https://docs.sentry.io/platforms/ruby/guides/rails/configuration/filtering/#using-beforesend + def self.call(event) + # Raven::Event instances don't need this processing. + return event unless event.is_a?(Sentry::Event) + + if event.request.present? + event.request.cookies = {} + event.request.data = {} + end + + if event.request.present? && event.request.headers.is_a?(Hash) + header_filter = ActiveSupport::ParameterFilter.new(SANITIZED_HTTP_HEADERS) + event.request.headers = header_filter.filter(event.request.headers) + end + + attribute_filter = ActiveSupport::ParameterFilter.new(Rails.application.config.filter_parameters) + SANITIZED_ATTRIBUTES.each do |attribute| + event.send("#{attribute}=", attribute_filter.filter(event.send(attribute))) # rubocop:disable GitlabSecurity/PublicSend + end + + if event.request.present? && event.request.query_string.present? + query = Rack::Utils.parse_nested_query(event.request.query_string) + query = attribute_filter.filter(query) + query = Rack::Utils.build_nested_query(query) + event.request.query_string = query + end + + event + end + end + end + end +end diff --git a/lib/gitlab/event_store.rb b/lib/gitlab/event_store.rb index e20ea1c7365..4955e873688 100644 --- a/lib/gitlab/event_store.rb +++ b/lib/gitlab/event_store.rb @@ -35,6 +35,11 @@ module Gitlab store.subscribe ::MergeRequests::UpdateHeadPipelineWorker, to: ::Ci::PipelineCreatedEvent store.subscribe ::Namespaces::UpdateRootStatisticsWorker, to: ::Projects::ProjectDeletedEvent + + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Pages::PageDeployedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Pages::PageDeletedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectDeletedEvent + store.subscribe ::Pages::InvalidateDomainCacheWorker, to: ::Projects::ProjectCreatedEvent end private_class_method :configure! end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 505d0b8d728..882bd57eb1d 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -18,6 +18,8 @@ module Gitlab UnknownRef = Class.new(BaseError) CommandTimedOut = Class.new(CommandError) InvalidPageToken = Class.new(BaseError) + InvalidRefFormatError = Class.new(BaseError) + ReferencesLockedError = Class.new(BaseError) class << self include Gitlab::EncodingHelper diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index f72217dedde..bb5bbeeb27e 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -128,6 +128,9 @@ module Gitlab @loaded_size = @data.bytesize if @data @loaded_all_data = @loaded_size == size + # Recalculate binary status if we loaded all data + @binary = nil if @loaded_all_data + record_metric_blob_size record_metric_truncated(truncated?) end diff --git a/lib/gitlab/git/conflict/parser.rb b/lib/gitlab/git/conflict/parser.rb index 20de8ebde4e..670305de95b 100644 --- a/lib/gitlab/git/conflict/parser.rb +++ b/lib/gitlab/git/conflict/parser.rb @@ -27,7 +27,7 @@ module Gitlab conflict_end = ">>>>>>> #{their_path}" text.each_line.map do |line| - full_line = line.delete("\n") + full_line = line.chomp if full_line == conflict_start validate_delimiter!(type.nil?) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index df744bd60b4..d7f892ae9d9 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -800,9 +800,9 @@ module Gitlab end end - def list_refs + def list_refs(patterns = [Gitlab::Git::BRANCH_REF_PREFIX]) wrapped_gitaly_errors do - gitaly_ref_client.list_refs + gitaly_ref_client.list_refs(patterns) end end diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index 40c003821b9..bc0af12d7e3 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -63,10 +63,7 @@ module Gitlab def tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive) tree_entries_from_rugged(repository, sha, path, recursive).tap do |entries| # This was an optimization to reduce N+1 queries for Gitaly - # (https://gitlab.com/gitlab-org/gitaly/issues/530). It - # used to be done lazily in the view via - # TreeHelper#flatten_tree, so it's possible there's a - # performance impact by loading this eagerly. + # (https://gitlab.com/gitlab-org/gitaly/issues/530). rugged_populate_flat_path(repository, sha, path, entries) end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index f376dbce177..996534f4194 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -285,6 +285,8 @@ module Gitlab end def self.enforce_gitaly_request_limits? + return false if ENV["GITALY_DISABLE_REQUEST_LIMITS"] + # We typically don't want to enforce request limits in production # However, we have some production-like test environments, i.e., ones # where `Rails.env.production?` returns `true`. We do want to be able to @@ -293,7 +295,7 @@ module Gitlab # enforce request limits. return true if Feature::Gitaly.enabled?('enforce_requests_limits') - !(Rails.env.production? || ENV["GITALY_DISABLE_REQUEST_LIMITS"]) + !Rails.env.production? end private_class_method :enforce_gitaly_request_limits? @@ -483,6 +485,22 @@ module Gitlab stack_counter.select { |_, v| v == max }.keys end + + def self.decode_detailed_error(err) + # details could have more than one in theory, but we only have one to worry about for now. + detailed_error = err.to_rpc_status&.details&.first + + return unless detailed_error.present? + + prefix = %r{type\.googleapis\.com\/gitaly\.(?<error_type>.+)} + error_type = prefix.match(detailed_error.type_url)[:error_type] + + Gitaly.const_get(error_type, false).decode(detailed_error.value) + rescue NameError, NoMethodError + # Error Class might not be known to ruby yet + nil + end + private_class_method :max_stacks end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index d575c0f470d..35d3ddf5d7f 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -102,7 +102,7 @@ module Gitlab raise Gitlab::Git::PreReceiveError, pre_receive_error end rescue GRPC::BadStatus => e - detailed_error = decode_detailed_error(e) + detailed_error = GitalyClient.decode_detailed_error(e) case detailed_error&.error when :custom_hook @@ -166,7 +166,7 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) rescue GRPC::BadStatus => e - detailed_error = decode_detailed_error(e) + detailed_error = GitalyClient.decode_detailed_error(e) case detailed_error&.error when :access_check @@ -277,7 +277,7 @@ module Gitlab rebase_sha rescue GRPC::BadStatus => e - detailed_error = decode_detailed_error(e) + detailed_error = GitalyClient.decode_detailed_error(e) case detailed_error&.error when :access_check @@ -314,7 +314,7 @@ module Gitlab response.squash_sha rescue GRPC::BadStatus => e - detailed_error = decode_detailed_error(e) + detailed_error = GitalyClient.decode_detailed_error(e) case detailed_error&.error when :resolve_revision, :rebase_conflict @@ -474,7 +474,7 @@ module Gitlab handle_cherry_pick_or_revert_response(response) rescue GRPC::BadStatus => e - detailed_error = decode_detailed_error(e) + detailed_error = GitalyClient.decode_detailed_error(e) case detailed_error&.error when :access_check @@ -483,6 +483,8 @@ module Gitlab raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) when :cherry_pick_conflict raise Gitlab::Git::Repository::CreateTreeError, 'CONFLICT' + when :changes_already_applied + raise Gitlab::Git::Repository::CreateTreeError, 'EMPTY' when :target_branch_diverged raise Gitlab::Git::CommitError, 'branch diverged' else @@ -536,21 +538,6 @@ module Gitlab raise ArgumentError, "Unknown action '#{action[:action]}'" end - def decode_detailed_error(err) - # details could have more than one in theory, but we only have one to worry about for now. - detailed_error = err.to_rpc_status&.details&.first - - return unless detailed_error.present? - - prefix = %r{type\.googleapis\.com\/gitaly\.(?<error_type>.+)} - error_type = prefix.match(detailed_error.type_url)[:error_type] - - Gitaly.const_get(error_type, false).decode(detailed_error.value) - rescue NameError, NoMethodError - # Error Class might not be known to ruby yet - nil - end - def custom_hook_error_message(custom_hook_error) # Custom hooks may return messages via either stdout or stderr which have a specific prefix. If # that prefix is present we'll want to print the hook's output, otherwise we'll want to print the diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index c064811b1e7..31e1406356f 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -132,6 +132,17 @@ module Gitlab response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.medium_timeout) raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error&.error + when :invalid_format + raise Gitlab::Git::InvalidRefFormatError, "references have an invalid format: #{detailed_error.invalid_format.refs.join(",")}" + when :references_locked + raise Gitlab::Git::ReferencesLockedError + else + raise e + end end # Limit: 0 implies no limit, thus all tag names will be returned diff --git a/lib/gitlab/github_import/importer/events/changed_label.rb b/lib/gitlab/github_import/importer/events/changed_label.rb new file mode 100644 index 00000000000..6c408158b02 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/changed_label.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class ChangedLabel + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + create_event(issue_event) + end + + private + + attr_reader :project, :user_id + + def create_event(issue_event) + ResourceLabelEvent.create!( + issue_id: issue_event.issue_db_id, + user_id: user_id, + label_id: label_finder.id_for(issue_event.label_title), + action: action(issue_event.event), + created_at: issue_event.created_at + ) + end + + def label_finder + Gitlab::GithubImport::LabelFinder.new(project) + end + + def action(event_type) + event_type == 'unlabeled' ? 'remove' : 'add' + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/closed.rb b/lib/gitlab/github_import/importer/events/closed.rb new file mode 100644 index 00000000000..8b2136c9b24 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/closed.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Closed + attr_reader :project, :user_id + + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + create_event(issue_event) + create_state_event(issue_event) + end + + private + + def create_event(issue_event) + Event.create!( + project_id: project.id, + author_id: user_id, + action: 'closed', + target_type: Issue.name, + target_id: issue_event.issue_db_id, + created_at: issue_event.created_at, + updated_at: issue_event.created_at + ) + end + + def create_state_event(issue_event) + ResourceStateEvent.create!( + user_id: user_id, + issue_id: issue_event.issue_db_id, + source_commit: issue_event.commit_id, + state: 'closed', + close_after_error_tracking_resolve: false, + close_auto_resolve_prometheus_alert: false, + created_at: issue_event.created_at + ) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb new file mode 100644 index 00000000000..20b902cfe50 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class CrossReferenced + attr_reader :project, :user_id + + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + mentioned_in_record_class = mentioned_in_type(issue_event) + mentioned_in_number = issue_event.source.dig(:issue, :number) + mentioned_in_record = init_mentioned_in( + mentioned_in_record_class, mentioned_in_number + ) + return if mentioned_in_record.nil? + + note_body = cross_reference_note_content(mentioned_in_record.gfm_reference(project)) + track_activity(mentioned_in_record_class) + create_note(issue_event, note_body) + end + + private + + def track_activity(mentioned_in_class) + return if mentioned_in_class != Issue + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event( + Gitlab::UsageDataCounters::IssueActivityUniqueCounter::ISSUE_CROSS_REFERENCED, + values: user_id + ) + end + + def create_note(issue_event, note_body) + Note.create!( + system: true, + noteable_type: Issue.name, + noteable_id: issue_event.issue_db_id, + project: project, + author_id: user_id, + note: note_body, + system_note_metadata: SystemNoteMetadata.new(action: 'cross_reference'), + created_at: issue_event.created_at + ) + end + + def mentioned_in_type(issue_event) + is_pull_request = issue_event.source.dig(:issue, :pull_request).present? + is_pull_request ? MergeRequest : Issue + end + + # record_class - Issue/MergeRequest + def init_mentioned_in(record_class, iid) + db_id = fetch_mentioned_in_db_id(record_class, iid) + return if db_id.nil? + + record = record_class.new(id: db_id, iid: iid) + record.project = project + record.readonly! + record + end + + # record_class - Issue/MergeRequest + def fetch_mentioned_in_db_id(record_class, number) + sawyer_mentioned_in_adapter = Struct.new(:iid, :issuable_type, keyword_init: true) + mentioned_in_adapter = sawyer_mentioned_in_adapter.new( + iid: number, issuable_type: record_class.name + ) + + Gitlab::GithubImport::IssuableFinder.new(project, mentioned_in_adapter).database_id + end + + def cross_reference_note_content(gfm_reference) + "#{::SystemNotes::IssuablesService.cross_reference_note_prefix}#{gfm_reference}" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/renamed.rb b/lib/gitlab/github_import/importer/events/renamed.rb new file mode 100644 index 00000000000..6a11c492210 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/renamed.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Renamed + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent` + def execute(issue_event) + Note.create!(note_params(issue_event)) + end + + private + + attr_reader :project, :user_id + + def note_params(issue_event) + { + noteable_id: issue_event.issue_db_id, + noteable_type: Issue.name, + project_id: project.id, + author_id: user_id, + note: parse_body(issue_event), + system: true, + created_at: issue_event.created_at, + updated_at: issue_event.created_at, + system_note_metadata: SystemNoteMetadata.new( + { + action: "title", + created_at: issue_event.created_at, + updated_at: issue_event.created_at + } + ) + } + end + + def parse_body(issue_event) + old_diffs, new_diffs = Gitlab::Diff::InlineDiff.new( + issue_event.old_title, issue_event.new_title + ).inline_diffs + + marked_old_title = Gitlab::Diff::InlineDiffMarkdownMarker.new(issue_event.old_title).mark(old_diffs) + marked_new_title = Gitlab::Diff::InlineDiffMarkdownMarker.new(issue_event.new_title).mark(new_diffs) + + "changed title from **#{marked_old_title}** to **#{marked_new_title}**" + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/reopened.rb b/lib/gitlab/github_import/importer/events/reopened.rb new file mode 100644 index 00000000000..c0f3802bc46 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/reopened.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class Reopened + attr_reader :project, :user_id + + def initialize(project, user_id) + @project = project + @user_id = user_id + end + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + def execute(issue_event) + create_event(issue_event) + create_state_event(issue_event) + end + + private + + def create_event(issue_event) + Event.create!( + project_id: project.id, + author_id: user_id, + action: 'reopened', + target_type: Issue.name, + target_id: issue_event.issue_db_id, + created_at: issue_event.created_at, + updated_at: issue_event.created_at + ) + end + + def create_state_event(issue_event) + ResourceStateEvent.create!( + user_id: user_id, + issue_id: issue_event.issue_db_id, + state: 'reopened', + created_at: issue_event.created_at + ) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb new file mode 100644 index 00000000000..e451af61ec3 --- /dev/null +++ b/lib/gitlab/github_import/importer/issue_event_importer.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class IssueEventImporter + attr_reader :issue_event, :project, :client, :user_finder + + # issue_event - An instance of `Gitlab::GithubImport::Representation::IssueEvent`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(issue_event, project, client) + @issue_event = issue_event + @project = project + @client = client + @user_finder = UserFinder.new(project, client) + end + + def execute + case issue_event.event + when 'closed' + Gitlab::GithubImport::Importer::Events::Closed.new(project, author_id) + .execute(issue_event) + when 'reopened' + Gitlab::GithubImport::Importer::Events::Reopened.new(project, author_id) + .execute(issue_event) + when 'labeled', 'unlabeled' + Gitlab::GithubImport::Importer::Events::ChangedLabel.new(project, author_id) + .execute(issue_event) + when 'renamed' + Gitlab::GithubImport::Importer::Events::Renamed.new(project, author_id) + .execute(issue_event) + when 'cross-referenced' + Gitlab::GithubImport::Importer::Events::CrossReferenced.new(project, author_id) + .execute(issue_event) + else + Gitlab::GithubImport::Logger.debug( + message: 'UNSUPPORTED_EVENT_TYPE', + event_type: issue_event.event, event_github_id: issue_event.id + ) + end + end + + private + + def author_id + id, _status = user_finder.author_id_for(issue_event, author_key: :actor) + id + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index 35fd4bd88a0..e7d41856b04 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -51,6 +51,7 @@ module Gitlab title: issue.truncated_title, author_id: author_id, project_id: project.id, + namespace_id: project.project_namespace_id, description: description, milestone_id: milestone_finder.id_for(issue), state_id: ::Issue.available_states[issue.state], diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 673f56b5753..1410006af26 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -21,14 +21,12 @@ module Gitlab author_id, author_found = user_finder.author_id_for(note) - note_body = MarkdownText.format(note.note, note.author, author_found) - attributes = { noteable_type: note.noteable_type, noteable_id: noteable_id, project_id: project.id, author_id: author_id, - note: note_body, + note: note_body(author_found), discussion_id: note.discussion_id, system: false, created_at: note.created_at, @@ -48,6 +46,13 @@ module Gitlab def find_noteable_id GithubImport::IssuableFinder.new(project, note).database_id end + + private + + def note_body(author_found) + text = MarkdownText.convert_ref_links(note.note, project) + MarkdownText.format(text, note.author, author_found) + end end end end diff --git a/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb index a2c3d1bd057..e1d9ae44065 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_diff_notes_importer.rb @@ -37,15 +37,15 @@ module Gitlab private - def noteables - project.merge_requests.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord + def parent_collection + project.merge_requests.where.not(iid: already_imported_parents) # rubocop: disable CodeReuse/ActiveRecord end def page_counter_id(merge_request) "merge_request/#{merge_request.id}/#{collection_method}" end - def notes_imported_cache_key + def parent_imported_cache_key "github-importer/merge_request/diff_notes/already-imported/#{project.id}" end end diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb new file mode 100644 index 00000000000..45bbc25e637 --- /dev/null +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class SingleEndpointIssueEventsImporter + include ParallelScheduling + include SingleEndpointNotesImporting + + PROCESSED_PAGE_CACHE_KEY = 'issues/%{issue_iid}/%{collection}' + BATCH_SIZE = 100 + + def initialize(project, client, parallel: true) + @project = project + @client = client + @parallel = parallel + @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % + { project: project.id, collection: collection_method } + end + + def each_associated(parent_record, associated) + compose_associated_id!(parent_record, associated) + return if already_imported?(associated) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + associated.issue_db_id = parent_record.id + yield(associated) + + mark_as_imported(associated) + end + + def importer_class + IssueEventImporter + end + + def representation_class + Representation::IssueEvent + end + + def sidekiq_worker_class + ImportIssueEventWorker + end + + def object_type + :issue_event + end + + def collection_method + :issue_timeline + end + + def parent_collection + project.issues.where.not(iid: already_imported_parents).select(:id, :iid) # rubocop: disable CodeReuse/ActiveRecord + end + + def parent_imported_cache_key + "github-importer/issues/#{collection_method}/already-imported/#{project.id}" + end + + def page_counter_id(issue) + PROCESSED_PAGE_CACHE_KEY % { issue_iid: issue.iid, collection: collection_method } + end + + def id_for_already_imported_cache(event) + event.id + end + + def collection_options + { state: 'all', sort: 'created', direction: 'asc' } + end + + # Cross-referenced events on Github doesn't have id. + def compose_associated_id!(issue, event) + return if event.event != 'cross-referenced' + + event.id = "cross-reference##{issue.id}-in-#{event.source.issue.id}" + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb index 49569ed52d8..fe64df45700 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_notes_importer.rb @@ -37,15 +37,15 @@ module Gitlab private - def noteables - project.issues.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord + def parent_collection + project.issues.where.not(iid: already_imported_parents) # rubocop: disable CodeReuse/ActiveRecord end def page_counter_id(issue) "issue/#{issue.id}/#{collection_method}" end - def notes_imported_cache_key + def parent_imported_cache_key "github-importer/issue/notes/already-imported/#{project.id}" end end diff --git a/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb index d837639c14d..3b1991d2b88 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_merge_request_notes_importer.rb @@ -37,15 +37,15 @@ module Gitlab private - def noteables - project.merge_requests.where.not(iid: already_imported_noteables) # rubocop: disable CodeReuse/ActiveRecord + def parent_collection + project.merge_requests.where.not(iid: already_imported_parents) # rubocop: disable CodeReuse/ActiveRecord end def page_counter_id(merge_request) "merge_request/#{merge_request.id}/#{collection_method}" end - def notes_imported_cache_key + def parent_imported_cache_key "github-importer/merge_request/notes/already-imported/#{project.id}" end end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index 0b1c221bbec..692016bd005 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -5,8 +5,34 @@ module Gitlab class MarkdownText include Gitlab::EncodingHelper - def self.format(*args) - new(*args).to_s + ISSUE_REF_MATCHER = '%{github_url}/%{import_source}/issues' + PULL_REF_MATCHER = '%{github_url}/%{import_source}/pull' + + class << self + def format(*args) + new(*args).to_s + end + + # Links like `https://domain.github.com/<namespace>/<project>/pull/<iid>` needs to be converted + def convert_ref_links(text, project) + matcher_options = { github_url: github_url, import_source: project.import_source } + issue_ref_matcher = ISSUE_REF_MATCHER % matcher_options + pull_ref_matcher = PULL_REF_MATCHER % matcher_options + + url_helpers = Rails.application.routes.url_helpers + text.gsub(issue_ref_matcher, url_helpers.project_issues_url(project)) + .gsub(pull_ref_matcher, url_helpers.project_merge_requests_url(project)) + end + + private + + # Returns github domain without slash in the end + def github_url + oauth_config = Gitlab::Auth::OAuth::Provider.config_for('github') || {} + url = oauth_config['url'].presence || 'https://github.com' + url = url.chop if url.end_with?('/') + url + end end # text - The Markdown text as a String. diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb new file mode 100644 index 00000000000..9016338db3b --- /dev/null +++ b/lib/gitlab/github_import/representation/issue_event.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class IssueEvent + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :actor, :event, :commit_id, :label_title, :old_title, :new_title, + :source, :created_at + expose_attribute :issue_db_id # set in SingleEndpointIssueEventsImporter#each_associated + + # Builds a event from a GitHub API response. + # + # event - An instance of `Sawyer::Resource` containing the event details. + def self.from_api_response(event) + new( + id: event.id, + actor: event.actor && Representation::User.from_api_response(event.actor), + event: event.event, + commit_id: event.commit_id, + label_title: event.label && event.label[:name], + old_title: event.rename && event.rename[:from], + new_title: event.rename && event.rename[:to], + source: event.source, + issue_db_id: event.issue_db_id, + created_at: event.created_at + ) + end + + # Builds a event using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + hash = Representation.symbolize_hash(raw_hash) + hash[:actor] &&= Representation::User.from_json_hash(hash[:actor]) + + new(hash) + end + + # attributes - A Hash containing the event details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { id: id } + end + end + end + end +end diff --git a/lib/gitlab/github_import/single_endpoint_notes_importing.rb b/lib/gitlab/github_import/single_endpoint_notes_importing.rb index 43402ecd165..0a3559adde3 100644 --- a/lib/gitlab/github_import/single_endpoint_notes_importing.rb +++ b/lib/gitlab/github_import/single_endpoint_notes_importing.rb @@ -4,9 +4,14 @@ # - SingleEndpointDiffNotesImporter # - SingleEndpointIssueNotesImporter # - SingleEndpointMergeRequestNotesImporter +# if `github_importer_single_endpoint_notes_import` feature flag is on. # -# `github_importer_single_endpoint_notes_import` -# feature flag is on. +# - SingleEndpointIssueEventsImporter +# if `github_importer_issue_events_import` feature flag is on. +# +# Fetches associated objects page by page to each item of parent collection. +# Currently `associated` is note or event. +# Currently `parent` is MergeRequest or Issue record. # # It fetches 1 PR's associated objects at a time using `issue_comments` or # `pull_request_comments` endpoint, which is slower than `NotesImporter` @@ -18,67 +23,75 @@ module Gitlab module SingleEndpointNotesImporting BATCH_SIZE = 100 - def each_object_to_import - each_notes_page do |page| - page.objects.each do |note| - next if already_imported?(note) + def each_object_to_import(&block) + each_associated_page do |parent_record, associated_page| + associated_page.objects.each do |associated| + each_associated(parent_record, associated, &block) + end + end + end - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + def id_for_already_imported_cache(associated) + associated.id + end - yield(note) + def parent_collection + raise NotImplementedError + end - mark_as_imported(note) - end - end + def parent_imported_cache_key + raise NotImplementedError end - def id_for_already_imported_cache(note) - note.id + def page_counter_id(parent) + raise NotImplementedError end private - def each_notes_page - noteables.each_batch(of: BATCH_SIZE, column: :iid) do |batch| - batch.each do |noteable| - # The page counter needs to be scoped by noteable to avoid skipping - # pages of notes from already imported noteables. - page_counter = PageCounter.new(project, page_counter_id(noteable)) + # Sometimes we need to add some extra info from parent + # to associated record that is not available by default + # in Github API response object. For example: + # lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb:26 + def each_associated(_parent_record, associated) + return if already_imported?(associated) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + yield(associated) + + mark_as_imported(associated) + end + + def each_associated_page + parent_collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| + batch.each do |parent_record| + # The page counter needs to be scoped by parent_record to avoid skipping + # pages of notes from already imported parent_record. + page_counter = PageCounter.new(project, page_counter_id(parent_record)) repo = project.import_source options = collection_options.merge(page: page_counter.current) - client.each_page(collection_method, repo, noteable.iid, options) do |page| + client.each_page(collection_method, repo, parent_record.iid, options) do |page| next unless page_counter.set(page.number) - yield page + yield parent_record, page end - mark_notes_imported(noteable) + mark_parent_imported(parent_record) end end end - def mark_notes_imported(noteable) + def mark_parent_imported(parent) Gitlab::Cache::Import::Caching.set_add( - notes_imported_cache_key, - noteable.iid + parent_imported_cache_key, + parent.iid ) end - def already_imported_noteables - Gitlab::Cache::Import::Caching.values_from_set(notes_imported_cache_key) - end - - def noteables - NotImplementedError - end - - def notes_imported_cache_key - NotImplementedError - end - - def page_counter_id(noteable) - NotImplementedError + def already_imported_parents + Gitlab::Cache::Import::Caching.values_from_set(parent_imported_cache_key) end end end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 93483ee697a..efaa2ce3002 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -39,13 +39,9 @@ module Gitlab # # If the object has no author ID we'll use the ID of the GitLab ghost # user. - def author_id_for(object) - id = - if object&.author - user_id_for(object.author) - else - GithubImport.ghost_user_id - end + def author_id_for(object, author_key: :author) + user_info = author_key == :actor ? object&.actor : object&.author + id = user_info ? user_id_for(user_info) : GithubImport.ghost_user_id if id [id, true] diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index ab7de14b07a..a03aeb9c293 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -3,17 +3,6 @@ module Gitlab module Gpg class Commit < Gitlab::SignedCommit - def signature - super - - return @signature if @signature - - cached_signature = lazy_signature&.itself - return @signature = cached_signature if cached_signature.present? - - @signature = create_cached_signature! - end - def update_signature!(cached_signature) using_keychain do |gpg_key| cached_signature.update!(attributes(gpg_key)) @@ -23,12 +12,8 @@ module Gitlab private - def lazy_signature - BatchLoader.for(@commit.sha).batch do |shas, loader| - CommitSignatures::GpgSignature.by_commit_sha(shas).each do |signature| - loader.call(signature.commit_sha, signature) - end - end + def signature_class + CommitSignatures::GpgSignature end def using_keychain diff --git a/lib/gitlab/grape_logging/loggers/response_logger.rb b/lib/gitlab/grape_logging/loggers/response_logger.rb new file mode 100644 index 00000000000..0465f01f7f5 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/response_logger.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module GrapeLogging + module Loggers + class ResponseLogger < ::GrapeLogging::Loggers::Base + def parameters(_, response) + return {} unless Feature.enabled?(:log_response_length) + + response_bytes = 0 + response.each { |resp| response_bytes += resp.to_s.bytesize } + { + response_bytes: response_bytes + } + end + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index c284160e539..3e119a39e6d 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -199,7 +199,7 @@ module Gitlab field_name = field.try(:attribute_name) || field field_value = node[field_name] ordering[field_name] = if field_value.is_a?(Time) - field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') + field_value.to_s(:inspect) else field_value.to_s end diff --git a/lib/gitlab/harbor/client.rb b/lib/gitlab/harbor/client.rb index 06142ae2b40..ee40725ba95 100644 --- a/lib/gitlab/harbor/client.rb +++ b/lib/gitlab/harbor/client.rb @@ -21,14 +21,44 @@ module Gitlab { success: response.success? } end + def get_repositories(params) + get(url("projects/#{integration.project_name}/repositories"), params) + end + + def get_artifacts(params) + repository_name = params.delete(:repository_name) + get(url("projects/#{integration.project_name}/repositories/#{repository_name}/artifacts"), params) + end + + def get_tags(params) + repository_name = params.delete(:repository_name) + artifact_name = params.delete(:artifact_name) + get( + url("projects/#{integration.project_name}/repositories/#{repository_name}/artifacts/#{artifact_name}/tags"), + params + ) + end + private - def url(path) - Gitlab::Utils.append_path(base_url, path) + def get(path, params = {}) + options = { headers: headers, query: params } + response = Gitlab::HTTP.get(path, options) + + raise Gitlab::Harbor::Client::Error, 'request error' unless response.success? + + { + body: Gitlab::Json.parse(response.body), + total_count: response.headers['x-total-count'].to_i + } + rescue JSON::ParserError + raise Gitlab::Harbor::Client::Error, 'invalid response format' end - def base_url - Gitlab::Utils.append_path(integration.url, '/api/v2.0/') + # url must be used within get method otherwise this would avoid validation by GitLab::HTTP + def url(path) + base_url = Gitlab::Utils.append_path(integration.url, '/api/v2.0/') + Gitlab::Utils.append_path(base_url, path) end def headers diff --git a/lib/gitlab/harbor/query.rb b/lib/gitlab/harbor/query.rb new file mode 100644 index 00000000000..c120810ecf1 --- /dev/null +++ b/lib/gitlab/harbor/query.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +module Gitlab + module Harbor + class Query + include ActiveModel::Validations + + attr_reader :client, :repository_id, :artifact_id, :search, :limit, :sort, :page + + DEFAULT_LIMIT = 10 + SORT_REGEX = %r{\A(creation_time|update_time|name) (asc|desc)\z}.freeze + + validates :page, numericality: { greater_than: 0, integer: true }, allow_blank: true + validates :limit, numericality: { greater_than: 0, less_than_or_equal_to: 25, integer: true }, allow_blank: true + validates :repository_id, format: { + with: /\A[a-zA-Z0-9\_\.\-$]+\z/, + message: 'Id invalid' + }, allow_blank: true + validates :artifact_id, format: { + with: /\A[a-zA-Z0-9\_\.\-$]+\z/, + message: 'Id invalid' + }, allow_blank: true + validates :sort, format: { + with: SORT_REGEX, + message: 'params invalid' + }, allow_blank: true + validates :search, format: { + with: /\A([a-z\_]*=[a-zA-Z0-9\- :]*,*)*\z/, + message: 'params invalid' + }, allow_blank: true + + def initialize(integration, params) + @client = Client.new(integration) + @repository_id = params[:repository_id] + @artifact_id = params[:artifact_id] + @search = params[:search] + @limit = params[:limit] + @sort = params[:sort] + @page = params[:page] + validate + end + + def repositories + result = @client.get_repositories(query_options) + return [] if result[:total_count] == 0 + + Kaminari.paginate_array( + result[:body], + limit: query_page_size, + total_count: result[:total_count] + ) + end + + def artifacts + result = @client.get_artifacts(query_artifacts_options) + return [] if result[:total_count] == 0 + + Kaminari.paginate_array( + result[:body], + limit: query_page_size, + total_count: result[:total_count] + ) + end + + def tags + result = @client.get_tags(query_tags_options) + return [] if result[:total_count] == 0 + + Kaminari.paginate_array( + result[:body], + limit: query_page_size, + total_count: result[:total_count] + ) + end + + private + + def query_artifacts_options + options = query_options + options[:repository_name] = repository_id + options[:with_tag] = true + + options + end + + def query_options + options = { + page: query_page, + page_size: query_page_size + } + + options[:q] = query_search if search.present? + options[:sort] = query_sort if sort.present? + + options + end + + def query_tags_options + options = query_options + options[:repository_name] = repository_id + options[:artifact_name] = artifact_id + + options + end + + def query_page + page.presence || 1 + end + + def query_page_size + (limit.presence || DEFAULT_LIMIT).to_i + end + + def query_search + search.gsub('=', '=~') + end + + def query_sort + match = sort.match(SORT_REGEX) + order = (match[2] == 'asc' ? '' : '-') + + "#{order}#{match[1]}" + end + end + end +end diff --git a/lib/gitlab/hash_digest/facade.rb b/lib/gitlab/hash_digest/facade.rb deleted file mode 100644 index d8efef02893..00000000000 --- a/lib/gitlab/hash_digest/facade.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HashDigest - # Used for rolling out to use OpenSSL::Digest::SHA256 - # for ActiveSupport::Digest - class Facade - class << self - def hexdigest(...) - hash_digest_class.hexdigest(...) - end - - def hash_digest_class - if use_sha256? - ::OpenSSL::Digest::SHA256 - else - ::Digest::MD5 # rubocop:disable Fips/MD5 - end - end - - def use_sha256? - return false unless Feature.feature_flags_available? - - Feature.enabled?(:active_support_hash_digest_sha256) - end - end - end - end -end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index 2b4bdbd48bd..b4f90715293 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -62,7 +62,8 @@ module Gitlab assignee_id: merge_request.assignee_ids.first, # This key is deprecated labels: merge_request.labels_hook_attrs, state: merge_request.state, # This key is deprecated - blocking_discussions_resolved: merge_request.mergeable_discussions_state? + blocking_discussions_resolved: merge_request.mergeable_discussions_state?, + first_contribution: merge_request.first_contribution? } merge_request.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index 7bb16e071b0..567c4dc899f 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -44,29 +44,19 @@ module Gitlab options end - options[:skip_read_total_timeout] = true if options[:skip_read_total_timeout].nil? && options[:stream_body] - - if options[:skip_read_total_timeout] + if options[:stream_body] return httparty_perform_request(http_method, path, options_with_timeouts, &block) end start_time = nil read_total_timeout = options.fetch(:timeout, DEFAULT_READ_TOTAL_TIMEOUT) - tracked_timeout_error = false httparty_perform_request(http_method, path, options_with_timeouts) do |fragment| start_time ||= Gitlab::Metrics::System.monotonic_time elapsed = Gitlab::Metrics::System.monotonic_time - start_time if elapsed > read_total_timeout - error = ReadTotalTimeout.new("Request timed out after #{elapsed} seconds") - - raise error if options[:use_read_total_timeout] - - unless tracked_timeout_error - Gitlab::ErrorTracking.track_exception(error) - tracked_timeout_error = true - end + raise ReadTotalTimeout, "Request timed out after #{elapsed} seconds" end block.call fragment if block diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 8d9f86d3232..cad0e773b05 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -25,6 +25,7 @@ module Gitlab 'pt_BR' => 'Portuguese (Brazil) - português (Brasil)', 'ro_RO' => 'Romanian - română', 'ru' => 'Russian - русский', + 'si_LK' => 'Sinhalese - සිංහල', 'tr_TR' => 'Turkish - Türkçe', 'uk' => 'Ukrainian - українська', 'zh_CN' => 'Chinese, Simplified - 简体中文', @@ -43,29 +44,30 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 41, - 'de' => 14, + 'da_DK' => 40, + 'de' => 15, 'en' => 100, 'eo' => 0, - 'es' => 36, + 'es' => 37, 'fil_PH' => 0, - 'fr' => 10, + 'fr' => 11, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, 'ja' => 33, - 'ko' => 12, - 'nb_NO' => 27, + 'ko' => 11, + 'nb_NO' => 26, 'nl_NL' => 0, 'pl_PL' => 4, - 'pt_BR' => 54, - 'ro_RO' => 79, - 'ru' => 29, + 'pt_BR' => 55, + 'ro_RO' => 100, + 'ru' => 28, + 'si_LK' => 11, 'tr_TR' => 12, - 'uk' => 44, - 'zh_CN' => 94, + 'uk' => 49, + 'zh_CN' => 99, 'zh_HK' => 2, - 'zh_TW' => 2 + 'zh_TW' => 4 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index ebabf537ce5..59396c6bad2 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -70,7 +70,11 @@ module Gitlab batch = batch.preload(key_preloads) if key_preloads batch.each do |record| + before_read_callback(record) + items << Raw.new(record.to_json(options)) + + after_read_callback(record) end end end @@ -168,6 +172,20 @@ module Gitlab def read_from_replica_if_available(&block) ::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries(&block) end + + def before_read_callback(record) + remove_cached_external_diff(record) + end + + def after_read_callback(record) + remove_cached_external_diff(record) + end + + def remove_cached_external_diff(record) + return unless record.is_a?(MergeRequest) + + record.merge_request_diff&.remove_cached_external_diff + end end end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 5a1787218f5..50ff6146174 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -761,6 +761,7 @@ excluded_attributes: - :exported_protected_branches - :repository_size_limit - :external_webhook_token + - :incident_management_issuable_escalation_statuses namespaces: - :runners_token - :runners_token_encrypted @@ -819,6 +820,7 @@ excluded_attributes: - :upvotes_count - :work_item_type_id - :email_message_id + - :incident_management_issuable_escalation_status merge_request: &merge_request_excluded_definition - :milestone_id - :sprint_id diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 8110720fb46..c4b0e24e34a 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -89,6 +89,7 @@ module Gitlab when :'Ci::PipelineSchedule' then setup_pipeline_schedule when :'ProtectedBranch::MergeAccessLevel' then setup_protected_branch_access_level when :'ProtectedBranch::PushAccessLevel' then setup_protected_branch_access_level + when :releases then setup_release end update_project_references @@ -133,7 +134,7 @@ module Gitlab end def setup_diff - diff = @relation_hash.delete('utf8_diff') + diff = @relation_hash.delete('diff_export') || @relation_hash.delete('utf8_diff') parsed_relation_hash['diff'] = diff end @@ -150,6 +151,14 @@ module Gitlab @relation_hash['relative_position'] = compute_relative_position end + def setup_release + # When author is not present for source release set the author as ghost user. + + if @relation_hash['author_id'].blank? + @relation_hash['author_id'] = User.select(:id).ghost.id + end + end + def setup_pipeline_schedule @relation_hash['active'] = false end diff --git a/lib/gitlab/issuable/clone/attributes_rewriter.rb b/lib/gitlab/issuable/clone/attributes_rewriter.rb new file mode 100644 index 00000000000..fd9b2f086fc --- /dev/null +++ b/lib/gitlab/issuable/clone/attributes_rewriter.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Issuable + module Clone + class AttributesRewriter + attr_reader :current_user, :original_entity, :target_parent + + def initialize(current_user, original_entity, target_parent) + raise ArgumentError, 'target_parent cannot be nil' if target_parent.nil? + + @current_user = current_user + @original_entity = original_entity + @target_parent = target_parent + end + + def execute(include_milestone: true) + attributes = { label_ids: cloneable_labels.pluck_primary_key } + + if include_milestone + milestone = matching_milestone(original_entity.milestone&.title) + attributes[:milestone_id] = milestone.id if milestone.present? + end + + attributes + end + + private + + def cloneable_labels + params = { + project_id: project&.id, + group_id: group&.id, + title: original_entity.labels.select(:title), + include_ancestor_groups: true + } + + params[:only_group_labels] = true if target_parent.is_a?(Group) + + LabelsFinder.new(current_user, params).execute + end + + def matching_milestone(title) + return if title.blank? + + params = { title: title, project_ids: project&.id, group_ids: group&.id } + + milestones = MilestonesFinder.new(params).execute + milestones.first + end + + def project + target_parent if target_parent.is_a?(Project) + end + + def group + if target_parent.is_a?(Group) + target_parent + elsif target_parent&.group && current_user.can?(:read_group, target_parent.group) + target_parent.group + end + end + end + end + end +end diff --git a/lib/gitlab/issuable/clone/copy_resource_events_service.rb b/lib/gitlab/issuable/clone/copy_resource_events_service.rb new file mode 100644 index 00000000000..563805fcb01 --- /dev/null +++ b/lib/gitlab/issuable/clone/copy_resource_events_service.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +module Gitlab + module Issuable + module Clone + class CopyResourceEventsService + attr_reader :current_user, :original_entity, :new_entity + + def initialize(current_user, original_entity, new_entity) + @current_user = current_user + @original_entity = original_entity + @new_entity = new_entity + end + + def execute + copy_resource_label_events + copy_resource_milestone_events + copy_resource_state_events + end + + private + + def copy_resource_label_events + copy_events(ResourceLabelEvent.table_name, original_entity.resource_label_events) do |event| + event.attributes + .except('id', 'reference', 'reference_html') + .merge(entity_key => new_entity.id, 'action' => ResourceLabelEvent.actions[event.action]) + end + end + + def copy_resource_milestone_events + return unless milestone_events_supported? + + copy_events(ResourceMilestoneEvent.table_name, original_entity.resource_milestone_events) do |event| + if event.remove? + event_attributes_with_milestone(event, nil) + else + destination_milestone = matching_milestone(event.milestone_title) + + event_attributes_with_milestone(event, destination_milestone) if destination_milestone.present? + end + end + end + + def copy_resource_state_events + return unless state_events_supported? + + copy_events(ResourceStateEvent.table_name, original_entity.resource_state_events) do |event| + event.attributes + .except(*blocked_state_event_attributes) + .merge(entity_key => new_entity.id, + 'state' => ResourceStateEvent.states[event.state]) + end + end + + # Overriden on EE::Gitlab::Issuable::Clone::CopyResourceEventsService + def blocked_state_event_attributes + ['id'] + end + + def event_attributes_with_milestone(event, milestone) + event.attributes + .except('id') + .merge(entity_key => new_entity.id, + 'milestone_id' => milestone&.id, + 'action' => ResourceMilestoneEvent.actions[event.action], + 'state' => ResourceMilestoneEvent.states[event.state]) + end + + def copy_events(table_name, events_to_copy) + events_to_copy.find_in_batches do |batch| + events = batch.map do |event| + yield(event) + end.compact + + ApplicationRecord.legacy_bulk_insert(table_name, events) # rubocop:disable Gitlab/BulkInsert + end + end + + def entity_key + new_entity.class.name.underscore.foreign_key + end + + def milestone_events_supported? + both_respond_to?(:resource_milestone_events) + end + + def state_events_supported? + both_respond_to?(:resource_state_events) + end + + def both_respond_to?(method) + original_entity.respond_to?(method) && + new_entity.respond_to?(method) + end + + def matching_milestone(title) + return if title.blank? || !new_entity.supports_milestone? + + params = { title: title, project_ids: new_entity.project&.id, group_ids: group&.id } + + milestones = MilestonesFinder.new(params).execute + milestones.first + end + + def group + if new_entity.project&.group && current_user.can?(:read_group, new_entity.project.group) + new_entity.project.group + end + end + end + end + end +end + +Gitlab::Issuable::Clone::CopyResourceEventsService.prepend_mod diff --git a/lib/gitlab/jira_import/issue_serializer.rb b/lib/gitlab/jira_import/issue_serializer.rb index ab748d67fbf..70ec6f08fcd 100644 --- a/lib/gitlab/jira_import/issue_serializer.rb +++ b/lib/gitlab/jira_import/issue_serializer.rb @@ -5,10 +5,11 @@ module Gitlab class IssueSerializer attr_reader :jira_issue, :project, :import_owner_id, :params, :formatter - def initialize(project, jira_issue, import_owner_id, params = {}) + def initialize(project, jira_issue, import_owner_id, work_item_type_id, params = {}) @jira_issue = jira_issue @project = project @import_owner_id = import_owner_id + @work_item_type_id = work_item_type_id @params = params @formatter = Gitlab::ImportFormatter.new end @@ -17,6 +18,7 @@ module Gitlab { iid: params[:iid], project_id: project.id, + namespace_id: project.project_namespace_id, description: description, title: title, state_id: map_status(jira_issue.status.statusCategory), @@ -24,7 +26,8 @@ module Gitlab created_at: jira_issue.created, author_id: reporter, assignee_ids: assignees, - label_ids: label_ids + label_ids: label_ids, + work_item_type_id: @work_item_type_id } end @@ -45,9 +48,9 @@ module Gitlab def map_status(jira_status_category) case jira_status_category["key"].downcase when 'done' - Issuable::STATE_ID_MAP[:closed] + ::Issuable::STATE_ID_MAP[:closed] else - Issuable::STATE_ID_MAP[:opened] + ::Issuable::STATE_ID_MAP[:opened] end end diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 8a03162f111..f1ead57c911 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -16,6 +16,7 @@ module Gitlab @start_at = Gitlab::JiraImport.get_issues_next_start_at(project.id) @imported_items_cache_key = JiraImport.already_imported_cache_key(:issues, project.id) @job_waiter = JobWaiter.new + @issue_type_id = WorkItems::Type.default_issue_type.id end def execute @@ -58,8 +59,13 @@ module Gitlab next if already_imported?(jira_issue.id) begin - issue_attrs = IssueSerializer.new(project, jira_issue, running_import.user_id, { iid: next_iid }).execute - + issue_attrs = IssueSerializer.new( + project, + jira_issue, + running_import.user_id, + @issue_type_id, + { iid: next_iid } + ).execute Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key) job_waiter.jobs_remaining += 1 diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 512936bb4f4..ce07752f88c 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -95,7 +95,7 @@ module Gitlab opts = standardize_opts(opts) Oj.load(string, opts) - rescue Oj::ParseError, Encoding::UndefinedConversionError => ex + rescue Oj::ParseError, EncodingError, Encoding::UndefinedConversionError => ex raise parser_error, ex end diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index 84ead5119d5..f8ec58cf217 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -7,7 +7,7 @@ module Gitlab LIMITED_ARRAY_SENTINEL = { key: 'truncated', value: '...' }.freeze IGNORE_PARAMS = Set.new(%w(controller action format)).freeze - KNOWN_PAYLOAD_PARAMS = [:remote_ip, :user_id, :username, :ua, :queue_duration_s, + KNOWN_PAYLOAD_PARAMS = [:remote_ip, :user_id, :username, :ua, :queue_duration_s, :response_bytes, :etag_route, :request_urgency, :target_duration_s] + CLOUDFLARE_CUSTOM_HEADERS.values def self.call(event) @@ -36,6 +36,10 @@ module Gitlab payload[:feature_flag_states] = Feature.logged_states.map { |key, state| "#{key}:#{state ? 1 : 0}" } end + if Feature.disabled?(:log_response_length) + payload.delete(:response_bytes) + end + payload end end diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb new file mode 100644 index 00000000000..db75ba8a47d --- /dev/null +++ b/lib/gitlab/memory/watchdog.rb @@ -0,0 +1,192 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + # A background thread that observes Ruby heap fragmentation and calls + # into a handler when the Ruby heap has been fragmented for an extended + # period of time. + # + # See Gitlab::Metrics::Memory for how heap fragmentation is defined. + # + # To decide whether a given fragmentation level is being exceeded, + # the watchdog regularly polls the GC. Whenever a violation occurs + # a strike is issued. If the maximum number of strikes are reached, + # a handler is invoked to deal with the situation. + # + # The duration for which a process may be above a given fragmentation + # threshold is computed as `max_strikes * sleep_time_seconds`. + class Watchdog < Daemon + DEFAULT_SLEEP_TIME_SECONDS = 60 + DEFAULT_HEAP_FRAG_THRESHOLD = 0.5 + DEFAULT_MAX_STRIKES = 5 + + # This handler does nothing. It returns `false` to indicate to the + # caller that the situation has not been dealt with so it will + # receive calls repeatedly if fragmentation remains high. + # + # This is useful for "dress rehearsals" in production since it allows + # us to observe how frequently the handler is invoked before taking action. + class NullHandler + include Singleton + + def on_high_heap_fragmentation(value) + # NOP + false + end + end + + # This handler sends SIGTERM and considers the situation handled. + class TermProcessHandler + def initialize(pid = $$) + @pid = pid + end + + def on_high_heap_fragmentation(value) + Process.kill(:TERM, @pid) + true + end + end + + # This handler invokes Puma's graceful termination handler, which takes + # into account a configurable grace period during which a process may + # remain unresponsive to a SIGTERM. + class PumaHandler + def initialize(puma_options = ::Puma.cli_config.options) + @worker = ::Puma::Cluster::WorkerHandle.new(0, $$, 0, puma_options) + end + + def on_high_heap_fragmentation(value) + @worker.term + true + end + end + + # max_heap_fragmentation: + # The degree to which the Ruby heap is allowed to be fragmented. Range [0,1]. + # max_strikes: + # How many times the process is allowed to be above max_heap_fragmentation before + # a handler is invoked. + # sleep_time_seconds: + # Used to control the frequency with which the watchdog will wake up and poll the GC. + def initialize( + handler: NullHandler.instance, + logger: Logger.new($stdout), + max_heap_fragmentation: ENV['GITLAB_MEMWD_MAX_HEAP_FRAG']&.to_f || DEFAULT_HEAP_FRAG_THRESHOLD, + max_strikes: ENV['GITLAB_MEMWD_MAX_STRIKES']&.to_i || DEFAULT_MAX_STRIKES, + sleep_time_seconds: ENV['GITLAB_MEMWD_SLEEP_TIME_SEC']&.to_i || DEFAULT_SLEEP_TIME_SECONDS, + **options) + super(**options) + + @handler = handler + @logger = logger + @max_heap_fragmentation = max_heap_fragmentation + @sleep_time_seconds = sleep_time_seconds + @max_strikes = max_strikes + + @alive = true + @strikes = 0 + + init_prometheus_metrics(max_heap_fragmentation) + end + + attr_reader :strikes, :max_heap_fragmentation, :max_strikes, :sleep_time_seconds + + def run_thread + @logger.info(log_labels.merge(message: 'started')) + + while @alive + sleep(@sleep_time_seconds) + + monitor_heap_fragmentation if Feature.enabled?(:gitlab_memory_watchdog, type: :ops) + end + + @logger.info(log_labels.merge(message: 'stopped')) + end + + private + + def monitor_heap_fragmentation + heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation + + if heap_fragmentation > @max_heap_fragmentation + @strikes += 1 + @heap_frag_violations.increment + else + @strikes = 0 + end + + if @strikes > @max_strikes + # If the handler returns true, it means the event is handled and we can shut down. + @alive = !handle_heap_fragmentation_limit_exceeded(heap_fragmentation) + @strikes = 0 + end + end + + def handle_heap_fragmentation_limit_exceeded(value) + @logger.warn( + log_labels.merge( + message: 'heap fragmentation limit exceeded', + memwd_cur_heap_frag: value + )) + @heap_frag_violations_handled.increment + + handler.on_high_heap_fragmentation(value) + end + + def handler + # This allows us to keep the watchdog running but turn it into "friendly mode" where + # all that happens is we collect logs and Prometheus events for fragmentation violations. + return NullHandler.instance unless Feature.enabled?(:enforce_memory_watchdog, type: :ops) + + @handler + end + + def stop_working + @alive = false + end + + def log_labels + { + pid: $$, + worker_id: worker_id, + memwd_handler_class: handler.class.name, + memwd_sleep_time_s: @sleep_time_seconds, + memwd_max_heap_frag: @max_heap_fragmentation, + memwd_max_strikes: @max_strikes, + memwd_cur_strikes: @strikes, + memwd_rss_bytes: process_rss_bytes + } + end + + def worker_id + ::Prometheus::PidProvider.worker_id + end + + def process_rss_bytes + Gitlab::Metrics::System.memory_usage_rss + end + + def init_prometheus_metrics(max_heap_fragmentation) + default_labels = { pid: worker_id } + + @heap_frag_limit = Gitlab::Metrics.gauge( + :gitlab_memwd_heap_frag_limit, + 'The configured limit for how fragmented the Ruby heap is allowed to be', + default_labels + ) + @heap_frag_limit.set({}, max_heap_fragmentation) + + @heap_frag_violations = Gitlab::Metrics.counter( + :gitlab_memwd_heap_frag_violations_total, + 'Total number of times heap fragmentation in a Ruby process exceeded its allowed maximum', + default_labels + ) + @heap_frag_violations_handled = Gitlab::Metrics.counter( + :gitlab_memwd_heap_frag_violations_handled_total, + 'Total number of times heap fragmentation violations in a Ruby process were handled', + default_labels + ) + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index ba2eb729d7b..858a0a120cc 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -7,6 +7,8 @@ module Gitlab module Metrics module Exporter class BaseExporter < Daemon + CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze + attr_reader :server # @param settings [Hash] SettingsLogic hash containing the `*_exporter` config @@ -38,10 +40,16 @@ module Gitlab [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT] ] - @server = ::WEBrick::HTTPServer.new( - Port: settings.port, BindAddress: settings.address, - Logger: logger, AccessLog: access_log - ) + server_config = { + Port: settings.port, + BindAddress: settings.address, + Logger: logger, + AccessLog: access_log + } + + server_config.merge!(ssl_config) if settings['tls_enabled'] + + @server = ::WEBrick::HTTPServer.new(server_config) server.mount '/', Rack::Handler::WEBrick, rack_app true @@ -82,6 +90,33 @@ module Gitlab run -> (env) { [404, {}, ['']] } end end + + def ssl_config + # This monkey-patches WEBrick::GenericServer, so never require this unless TLS is enabled. + require 'webrick/ssl' + + certs = load_ca_certs_bundle(File.binread(settings['tls_cert_path'])) + + { + SSLEnable: true, + SSLCertificate: certs.shift, + SSLPrivateKey: OpenSSL::PKey.read(File.binread(settings['tls_key_path'])), + # SSLStartImmediately is true by default according to the docs, but when WEBrick creates the + # SSLServer internally, the switch was always nil for some reason. Setting this explicitly fixes this. + SSLStartImmediately: true, + SSLExtraChainCert: certs + } + end + + # In Ruby OpenSSL v3.0.0, this can be replaced by OpenSSL::X509::Certificate.load + # https://github.com/ruby/openssl/issues/254 + def load_ca_certs_bundle(ca_certs_string) + return [] unless ca_certs_string + + ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string| + OpenSSL::X509::Certificate.new(ca_cert_string) + end + end end end end diff --git a/lib/gitlab/metrics/memory.rb b/lib/gitlab/metrics/memory.rb new file mode 100644 index 00000000000..c165cdec7a3 --- /dev/null +++ b/lib/gitlab/metrics/memory.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Memory + extend self + + HEAP_SLOTS_PER_PAGE = GC::INTERNAL_CONSTANTS[:HEAP_PAGE_OBJ_LIMIT] + + def gc_heap_fragmentation(gc_stat = GC.stat) + 1 - (gc_stat[:heap_live_slots] / (HEAP_SLOTS_PER_PAGE * gc_stat[:heap_eden_pages].to_f)) + end + end + end +end diff --git a/lib/gitlab/metrics/prometheus.rb b/lib/gitlab/metrics/prometheus.rb index 848b73792cb..0b513f5521e 100644 --- a/lib/gitlab/metrics/prometheus.rb +++ b/lib/gitlab/metrics/prometheus.rb @@ -29,6 +29,7 @@ module Gitlab clear_memoization(:registry) REGISTRY_MUTEX.synchronize do + ::Prometheus::Client.cleanup! ::Prometheus::Client.reset! end end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 4a3ef3711a5..8e002293347 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -39,7 +39,8 @@ module Gitlab process_proportional_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :proportional_memory_bytes), 'Memory used (PSS)', labels), process_start_time_seconds: ::Gitlab::Metrics.gauge(metric_name(:process, :start_time_seconds), 'Process start time seconds'), sampler_duration: ::Gitlab::Metrics.counter(metric_name(:sampler, :duration_seconds_total), 'Sampler time', labels), - gc_duration_seconds: ::Gitlab::Metrics.histogram(metric_name(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS) + gc_duration_seconds: ::Gitlab::Metrics.histogram(metric_name(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS), + heap_fragmentation: ::Gitlab::Metrics.gauge(metric_name(:gc_stat_ext, :heap_fragmentation), 'Ruby heap fragmentation', labels) } GC.stat.keys.each do |key| @@ -76,8 +77,13 @@ module Gitlab end # Collect generic GC stats - GC.stat.each do |key, value| - metrics[key].set(labels, value) + GC.stat.then do |gc_stat| + gc_stat.each do |key, value| + metrics[key].set(labels, value) + end + + # Collect custom GC stats + metrics[:heap_fragmentation].set(labels, Memory.gc_heap_fragmentation(gc_stat)) end end diff --git a/lib/gitlab/metrics/sli.rb b/lib/gitlab/metrics/sli.rb index 2de19514354..15cfe777f4d 100644 --- a/lib/gitlab/metrics/sli.rb +++ b/lib/gitlab/metrics/sli.rb @@ -82,7 +82,7 @@ module Gitlab private def counter_name(suffix) - :"#{COUNTER_PREFIX}:#{name}_apdex:#{suffix}" + [COUNTER_PREFIX, "#{name}_apdex", suffix].join('_').to_sym end def numerator_counter @@ -100,7 +100,7 @@ module Gitlab private def counter_name(suffix) - :"#{COUNTER_PREFIX}:#{name}:#{suffix}" + [COUNTER_PREFIX, name, suffix].join('_').to_sym end def numerator_counter diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 7c22ce64ea2..e3756a8c9f6 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -24,7 +24,7 @@ module Gitlab # This event is published from ActiveRecordBaseTransactionMetrics and # used to record a database transaction duration when calling - # ActiveRecord::Base.transaction {} block. + # ApplicationRecord.transaction {} block. def transaction(event) observe(:gitlab_database_transaction_seconds, event) do buckets TRANSACTION_DURATION_BUCKET @@ -186,7 +186,10 @@ module Gitlab end ::Gitlab::Database.database_base_models.keys.each do |config_name| - counters << compose_metric_key(metric, nil, config_name) # main / ci + counters << compose_metric_key(metric, nil, config_name) # main / ci / geo + end + + ::Gitlab::Database.database_base_models_using_load_balancing.keys.each do |config_name| counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica / ci_replica end end diff --git a/lib/gitlab/pages/cache_control.rb b/lib/gitlab/pages/cache_control.rb new file mode 100644 index 00000000000..991a1297d03 --- /dev/null +++ b/lib/gitlab/pages/cache_control.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class CacheControl + CACHE_KEY_FORMAT = 'pages_domain_for_%{type}_%{id}' + + attr_reader :cache_key + + class << self + def for_project(project_id) + new(type: :project, id: project_id) + end + + def for_namespace(namespace_id) + new(type: :namespace, id: namespace_id) + end + end + + def initialize(type:, id:) + raise(ArgumentError, "type must be :namespace or :project") unless %i[namespace project].include?(type) + + @cache_key = CACHE_KEY_FORMAT % { type: type, id: id } + end + + def clear_cache + Rails.cache.delete(cache_key) + end + end + end +end diff --git a/lib/gitlab/pages/deployment_update.rb b/lib/gitlab/pages/deployment_update.rb new file mode 100644 index 00000000000..2f5c6938e2a --- /dev/null +++ b/lib/gitlab/pages/deployment_update.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class DeploymentUpdate + include ActiveModel::Validations + + PUBLIC_DIR = 'public' + + validate :validate_state, unless: -> { errors.any? } + validate :validate_outdated_sha, unless: -> { errors.any? } + validate :validate_max_size, unless: -> { errors.any? } + validate :validate_public_folder, unless: -> { errors.any? } + validate :validate_max_entries, unless: -> { errors.any? } + + def initialize(project, build) + @project = project + @build = build + end + + def latest? + # check if sha for the ref is still the most recent one + # this helps in case when multiple deployments happens + sha == latest_sha + end + + def entries_count + # we're using the full archive and pages daemon needs to read it + # so we want the total count from entries, not only "public/" directory + # because it better approximates work we need to do before we can serve the site + @entries_count = build.artifacts_metadata_entry("", recursive: true).entries.count + end + + private + + attr_reader :build, :project + + def validate_state + errors.add(:base, 'missing pages artifacts') unless build.artifacts? + errors.add(:base, 'missing artifacts metadata') unless build.artifacts_metadata? + end + + def validate_max_size + if total_size > max_size + errors.add(:base, "artifacts for pages are too large: #{total_size}") + end + end + + # Calculate page size after extract + def total_size + @total_size ||= build.artifacts_metadata_entry(PUBLIC_DIR + '/', recursive: true).total_size + end + + def max_size_from_settings + Gitlab::CurrentSettings.max_pages_size.megabytes + end + + def max_size + max_pages_size = max_size_from_settings + + return ::Gitlab::Pages::MAX_SIZE if max_pages_size == 0 + + max_pages_size + end + + def validate_max_entries + if pages_file_entries_limit > 0 && entries_count > pages_file_entries_limit + errors.add( + :base, + "pages site contains #{entries_count} file entries, while limit is set to #{pages_file_entries_limit}" + ) + end + end + + def validate_public_folder + if total_size <= 0 + errors.add(:base, 'Error: The `public/` folder is missing, or not declared in `.gitlab-ci.yml`.') + end + end + + def pages_file_entries_limit + project.actual_limits.pages_file_entries + end + + def validate_outdated_sha + return if latest? + + # use pipeline_id in case the build is retried + last_deployed_pipeline_id = project.pages_metadatum&.pages_deployment&.ci_build&.pipeline_id + + return unless last_deployed_pipeline_id + return if last_deployed_pipeline_id <= build.pipeline_id + + errors.add(:base, 'build SHA is outdated for this ref') + end + + def latest_sha + project.commit(build.ref).try(:sha).to_s + ensure + # Close any file descriptors that were opened and free libgit2 buffers + project.cleanup + end + + def sha + build.sha + end + end + end +end + +Gitlab::Pages::DeploymentUpdate.prepend_mod_with('Gitlab::Pages::DeploymentUpdate') diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index f19cdf06d9a..2d9fb0a50fc 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -4,9 +4,18 @@ module Gitlab module Pagination module CursorBasedKeyset SUPPORTED_ORDERING = { - Group => { name: :asc } + Group => { name: :asc }, + AuditEvent => { id: :desc } }.freeze + # Relation types that are enforced in this list + # enforce the use of keyset pagination, thus erroring out requests + # made with offset pagination above a certain limit. + # + # In many cases this could introduce a breaking change + # so enforcement is optional. + ENFORCED_TYPES = [Group].freeze + def self.available_for_type?(relation) SUPPORTED_ORDERING.key?(relation.klass) end @@ -16,6 +25,10 @@ module Gitlab order_satisfied?(relation, cursor_based_request_context) end + def self.enforced_for_type?(relation) + ENFORCED_TYPES.include?(relation.klass) + end + def self.order_satisfied?(relation, cursor_based_request_context) order_by_from_request = cursor_based_request_context.order_by diff --git a/lib/gitlab/pagination/keyset/cursor_based_request_context.rb b/lib/gitlab/pagination/keyset/cursor_based_request_context.rb index e06d7e48ca3..41b90846345 100644 --- a/lib/gitlab/pagination/keyset/cursor_based_request_context.rb +++ b/lib/gitlab/pagination/keyset/cursor_based_request_context.rb @@ -5,6 +5,8 @@ module Gitlab module Keyset class CursorBasedRequestContext DEFAULT_SORT_DIRECTION = :desc + DEFAULT_SORT_COLUMN = :id + attr_reader :request_context delegate :params, to: :request_context @@ -28,7 +30,7 @@ module Gitlab end def order_by - { params[:order_by].to_sym => params[:sort]&.to_sym || DEFAULT_SORT_DIRECTION } + { (params[:order_by]&.to_sym || DEFAULT_SORT_COLUMN) => (params[:sort]&.to_sym || DEFAULT_SORT_DIRECTION) } end end end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index 290e94401b8..0d8e4ea6fee 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -96,7 +96,9 @@ module Gitlab column_definitions.each_with_object({}.with_indifferent_access) do |column_definition, hash| field_value = node[column_definition.attribute_name] hash[column_definition.attribute_name] = if field_value.is_a?(Time) - field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') + # use :inspect formatter to provide specific timezone info + # eg 2022-07-05 21:57:56.041499000 +0800 + field_value.to_s(:inspect) elsif field_value.nil? nil elsif lower_named_function?(column_definition) @@ -107,6 +109,10 @@ module Gitlab end end + def attribute_names + column_definitions.map(&:attribute_name) + end + # This methods builds the conditions for the keyset pagination # # Example: diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 259d9e38d65..3b85d6952a1 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -23,7 +23,7 @@ module Gitlab _('Closed this %{quick_action_target}.') % { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end - types Issuable + types ::Issuable condition do quick_action_target.persisted? && quick_action_target.open? && @@ -45,7 +45,7 @@ module Gitlab _('Reopened this %{quick_action_target}.') % { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end - types Issuable + types ::Issuable condition do quick_action_target.persisted? && quick_action_target.closed? && @@ -63,7 +63,7 @@ module Gitlab _('Changed the title to "%{title_param}".') % { title_param: title_param } end params '<New title>' - types Issuable + types ::Issuable condition do quick_action_target.persisted? && current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) @@ -82,7 +82,7 @@ module Gitlab end end params '~label1 ~"label 2"' - types Issuable + types ::Issuable condition do current_user.can?(:"set_#{quick_action_target.to_ability_name}_metadata", quick_action_target) && find_labels.any? @@ -102,7 +102,7 @@ module Gitlab end end params '~label1 ~"label 2"' - types Issuable + types ::Issuable condition do quick_action_target.persisted? && quick_action_target.labels.any? && @@ -134,7 +134,7 @@ module Gitlab "Replaces all labels with #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? end params '~label1 ~"label 2"' - types Issuable + types ::Issuable condition do quick_action_target.persisted? && quick_action_target.labels.any? && @@ -147,7 +147,7 @@ module Gitlab desc { _('Add a to do') } explanation { _('Adds a to do.') } execution_message { _('Added a to do.') } - types Issuable + types ::Issuable condition do quick_action_target.persisted? && !TodoService.new.todo_exist?(quick_action_target, current_user) @@ -159,7 +159,7 @@ module Gitlab desc { _('Mark to do as done') } explanation { _('Marks to do as done.') } execution_message { _('Marked to do as done.') } - types Issuable + types ::Issuable condition do quick_action_target.persisted? && TodoService.new.todo_exist?(quick_action_target, current_user) @@ -177,7 +177,7 @@ module Gitlab _('Subscribed to this %{quick_action_target}.') % { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end - types Issuable + types ::Issuable condition do quick_action_target.persisted? && !quick_action_target.subscribed?(current_user, project) @@ -195,7 +195,7 @@ module Gitlab _('Unsubscribed from this %{quick_action_target}.') % { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } end - types Issuable + types ::Issuable condition do quick_action_target.persisted? && quick_action_target.subscribed?(current_user, project) @@ -212,7 +212,7 @@ module Gitlab _("Toggled :%{name}: emoji award.") % { name: name } if name end params ':emoji:' - types Issuable + types ::Issuable condition do quick_action_target.persisted? end @@ -228,14 +228,14 @@ module Gitlab desc { _("Append the comment with %{shrug}") % { shrug: SHRUG } } params '<Comment>' - types Issuable + types ::Issuable substitution :shrug do |comment| "#{comment} #{SHRUG}" end desc { _("Append the comment with %{tableflip}") % { tableflip: TABLEFLIP } } params '<Comment>' - types Issuable + types ::Issuable substitution :tableflip do |comment| "#{comment} #{TABLEFLIP}" end diff --git a/lib/gitlab/quick_actions/users_extractor.rb b/lib/gitlab/quick_actions/users_extractor.rb new file mode 100644 index 00000000000..06e04c74312 --- /dev/null +++ b/lib/gitlab/quick_actions/users_extractor.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Gitlab + module QuickActions + class UsersExtractor + MAX_QUICK_ACTION_USERS = 100 + + Error = Class.new(ArgumentError) + TooManyError = Class.new(Error) do + def limit + MAX_QUICK_ACTION_USERS + end + end + + MissingError = Class.new(Error) + TooManyFoundError = Class.new(TooManyError) + TooManyRefsError = Class.new(TooManyError) + + attr_reader :text, :current_user, :project, :group, :target + + def initialize(current_user, project:, group:, target:, text:) + @current_user = current_user + @project = project + @group = group + @target = target + @text = text + end + + def execute + return [] unless text.present? + + users = collect_users + + check_users!(users) + + users + end + + private + + def collect_users + users = [] + users << current_user if me? + users += find_referenced_users if references.any? + + users + end + + def check_users!(users) + raise TooManyFoundError if users.size > MAX_QUICK_ACTION_USERS + + found = found_names(users) + missing = references.filter_map do + "'#{_1}'" unless found.include?(_1.downcase.delete_prefix('@')) + end + + raise MissingError, missing.to_sentence if missing.present? + end + + def found_names(users) + users.map(&:username).map(&:downcase).to_set + end + + def find_referenced_users + raise TooManyRefsError if references.size > MAX_QUICK_ACTION_USERS + + User.by_username(usernames).limit(MAX_QUICK_ACTION_USERS) + end + + def usernames + references.map { _1.delete_prefix('@') } + end + + def references + @references ||= begin + refs = args - ['me'] + # nb: underscores may be passed in escaped to protect them from markdown rendering + refs.map! { _1.gsub(/\\_/, '_') } + refs + end + end + + def args + @args ||= text.split(/\s|,/).map(&:strip).select(&:present?).uniq - ['and'] + end + + def me? + args&.include?('me') + end + end + end +end + +Gitlab::QuickActions::UsersExtractor.prepend_mod_with('Gitlab::QuickActions::UsersExtractor') diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index 24c540eea47..94f06e957cf 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -11,8 +11,15 @@ module Gitlab end end class PipelinedDiffError < StandardError + def initialize(result_primary, result_secondary) + @result_primary = result_primary + @result_secondary = result_secondary + end + def message - 'Pipelined command executed on both stores successfully but results differ between them.' + "Pipelined command executed on both stores successfully but results differ between them. " \ + "Result from the primary: #{@result_primary.inspect}. " \ + "Result from the secondary: #{@result_secondary.inspect}." end end class MethodMissingError < StandardError @@ -246,10 +253,12 @@ module Gitlab result_secondary = send_command(secondary_store, command_name, *args, **kwargs, &block) - # Pipelined commands return an array with all results. If they differ, - # log an error - if result_primary != result_secondary - log_error(PipelinedDiffError.new, command_name) + # Pipelined commands return an array with all results. If they differ, log an error + if result_primary && result_primary != result_secondary + error = PipelinedDiffError.new(result_primary, result_secondary) + error.set_backtrace(Thread.current.backtrace[1..]) # Manually set backtrace, since the error is not `raise`d + + log_error(error, command_name) increment_pipelined_command_error_count(command_name) end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index b0f4194b7a0..0534f890152 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -486,6 +486,10 @@ module Gitlab def sep_by_1(separator, part) %r(#{part} (#{separator} #{part})*)x end + + def x509_subject_key_identifier_regex + @x509_subject_key_identifier_regex ||= /\A(?:\h{2}:)*\h{2}\z/.freeze + end end end diff --git a/lib/gitlab/saas.rb b/lib/gitlab/saas.rb index 0a4f2ba64a8..4683f611444 100644 --- a/lib/gitlab/saas.rb +++ b/lib/gitlab/saas.rb @@ -48,6 +48,10 @@ module Gitlab def self.about_pricing_faq_url "https://about.gitlab.com/pricing#faq" end + + def self.doc_url + 'https://docs.gitlab.com' + end end end diff --git a/lib/gitlab/security/scan_configuration.rb b/lib/gitlab/security/scan_configuration.rb index 14883a34950..9b09ccdeb8e 100644 --- a/lib/gitlab/security/scan_configuration.rb +++ b/lib/gitlab/security/scan_configuration.rb @@ -18,7 +18,7 @@ module Gitlab # SAST and Secret Detection are always available, but this isn't # reflected by our license model yet. # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/333113 - %i[sast sast_iac secret_detection].include?(type) + %i[sast sast_iac secret_detection container_scanning].include?(type) end def can_enable_by_merge_request? diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index 113076a6a75..cb7d9c6f8a7 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -145,6 +145,8 @@ module Gitlab sleep(CHECK_INTERVAL_SECONDS) refresh_state(:above_soft_limit) + + log_rss_out_of_range(false) end # There are two chances to break from loop: @@ -153,28 +155,49 @@ module Gitlab # When `above hard limit`, it immediately go to `stop_fetching_new_jobs` # So ignore `above hard limit` and always set `above_soft_limit` here refresh_state(:above_soft_limit) - log_rss_out_of_range(@current_rss, @hard_limit_rss, @soft_limit_rss) + log_rss_out_of_range false end - def log_rss_out_of_range(current_rss, hard_limit_rss, soft_limit_rss) + def log_rss_out_of_range(deadline_exceeded = true) + reason = out_of_range_description(@current_rss, + @hard_limit_rss, + @soft_limit_rss, + deadline_exceeded) + Sidekiq.logger.warn( class: self.class.to_s, pid: pid, message: 'Sidekiq worker RSS out of range', - current_rss: current_rss, - hard_limit_rss: hard_limit_rss, - soft_limit_rss: soft_limit_rss, - reason: out_of_range_description(current_rss, hard_limit_rss, soft_limit_rss) - ) + current_rss: @current_rss, + soft_limit_rss: @soft_limit_rss, + hard_limit_rss: @hard_limit_rss, + reason: reason, + running_jobs: running_jobs) end - def out_of_range_description(rss, hard_limit, soft_limit) + def running_jobs + jobs = [] + Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do + jobs = Gitlab::SidekiqDaemon::Monitor.instance.jobs.map do |jid, job| + { + jid: jid, + worker_class: job[:worker_class].name + } + end + end + + jobs + end + + def out_of_range_description(rss, hard_limit, soft_limit, deadline_exceeded) if rss > hard_limit "current_rss(#{rss}) > hard_limit_rss(#{hard_limit})" - else + elsif deadline_exceeded "current_rss(#{rss}) > soft_limit_rss(#{soft_limit}) longer than GRACE_BALLOON_SECONDS(#{GRACE_BALLOON_SECONDS})" + else + "current_rss(#{rss}) > soft_limit_rss(#{soft_limit})" end end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 6eb39981ef4..7ce3f6b5ccb 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -81,12 +81,6 @@ module Gitlab payload['job_status'] = 'fail' Gitlab::ExceptionLogFormatter.format!(job_exception, payload) - - # Deprecated fields for compatibility - # See https://gitlab.com/gitlab-org/gitlab/-/issues/364241 - payload['error_class'] = payload['exception.class'] - payload['error_message'] = payload['exception.message'] - payload['error_backtrace'] = payload['exception.backtrace'] else payload['message'] = "#{message}: done: #{payload['duration_s']} sec" payload['job_status'] = 'done' diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index dc5481289da..ea2b405c934 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -31,6 +31,7 @@ module Gitlab sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_jobs_interrupted_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_interrupted_total, 'Sidekiq jobs interrupted'), sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'), sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), @@ -89,6 +90,10 @@ module Gitlab @metrics[:sidekiq_jobs_retried_total].increment(labels, 1) end + if job['interrupted_count'].present? + @metrics[:sidekiq_jobs_interrupted_total].increment(labels, 1) + end + job_succeeded = false monotonic_time_start = Gitlab::Metrics::System.monotonic_time job_thread_cputime_start = get_thread_cputime diff --git a/lib/gitlab/signed_commit.rb b/lib/gitlab/signed_commit.rb index 7a154978938..410e71f51a1 100644 --- a/lib/gitlab/signed_commit.rb +++ b/lib/gitlab/signed_commit.rb @@ -18,7 +18,18 @@ module Gitlab end def signature - return unless @commit.has_signature? + return @signature if @signature + + cached_signature = lazy_signature&.itself + + return @signature = cached_signature if cached_signature.present? + + @signature = create_cached_signature! + end + + def update_signature!(cached_signature) + cached_signature.update!(attributes) + @signature = cached_signature end def signature_text @@ -32,5 +43,27 @@ module Gitlab @signature_data.itself ? @signature_data[1] : nil end end + + private + + def signature_class + raise NotImplementedError, '`signature_class` must be implmented by subclass`' + end + + def lazy_signature + BatchLoader.for(@commit.sha).batch do |shas, loader| + signature_class.by_commit_sha(shas).each do |signature| + loader.call(signature.commit_sha, signature) + end + end + end + + def create_cached_signature! + return if attributes.nil? + + return signature_class.new(attributes) if Gitlab::Database.read_only? + + signature_class.safe_create!(attributes) + end end end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 0e7812d08b8..04745bafe7c 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -4,13 +4,13 @@ module Gitlab module Tracking class << self def enabled? - snowplow.enabled? + tracker.enabled? end def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace, **extra).to_context, *context] - snowplow.event(category, action, label: label, property: property, value: value, context: contexts) + tracker.event(category, action, label: label, property: property, value: value, context: contexts) rescue StandardError => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) end @@ -31,25 +31,27 @@ module Gitlab end def options(group) - snowplow.options(group) + tracker.options(group) end def collector_hostname - snowplow.hostname + tracker.hostname end def snowplow_micro_enabled? - Rails.env.development? && Gitlab::Utils.to_boolean(ENV['SNOWPLOW_MICRO_ENABLE']) + Rails.env.development? && Gitlab.config.snowplow_micro.enabled + rescue Settingslogic::MissingSetting + Gitlab::Utils.to_boolean(ENV['SNOWPLOW_MICRO_ENABLE']) end private - def snowplow - @snowplow ||= if snowplow_micro_enabled? - Gitlab::Tracking::Destinations::SnowplowMicro.new - else - Gitlab::Tracking::Destinations::Snowplow.new - end + def tracker + @tracker ||= if snowplow_micro_enabled? + Gitlab::Tracking::Destinations::SnowplowMicro.new + else + Gitlab::Tracking::Destinations::Snowplow.new + end end end end diff --git a/lib/gitlab/tracking/destinations/snowplow_micro.rb b/lib/gitlab/tracking/destinations/snowplow_micro.rb index 3553efba1e1..c7a95e88d0b 100644 --- a/lib/gitlab/tracking/destinations/snowplow_micro.rb +++ b/lib/gitlab/tracking/destinations/snowplow_micro.rb @@ -30,8 +30,9 @@ module Gitlab def uri strong_memoize(:snowplow_uri) do - uri = URI(ENV['SNOWPLOW_MICRO_URI'] || DEFAULT_URI) - uri = URI("http://#{ENV['SNOWPLOW_MICRO_URI']}") unless %w[http https].include?(uri.scheme) + base = base_uri + uri = URI(base) + uri = URI("http://#{base}") unless %w[http https].include?(uri.scheme) uri end end @@ -47,6 +48,14 @@ module Gitlab def protocol uri.scheme end + + def base_uri + url = Gitlab.config.snowplow_micro.address + scheme = Gitlab.config.gitlab.https ? 'https' : 'http' + "#{scheme}://#{url}" + rescue Settingslogic::MissingSetting + ENV['SNOWPLOW_MICRO_URI'] || DEFAULT_URI + end end end end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 05ddc7e26cc..50467de44b8 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -7,11 +7,9 @@ module Gitlab GITLAB_RAILS_SOURCE = 'gitlab-rails' def initialize(namespace: nil, project: nil, user: nil, **extra) - if Feature.enabled?(:standard_context_type_check) - check_argument_type(:namespace, namespace, [Namespace]) - check_argument_type(:project, project, [Project, Integer]) - check_argument_type(:user, user, [User, DeployToken]) - end + check_argument_type(:namespace, namespace, [Namespace]) + check_argument_type(:project, project, [Project, Integer]) + check_argument_type(:user, user, [User, DeployToken]) @namespace = namespace @plan = namespace&.actual_plan_name diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index 85f0ba1fd25..72df8b423df 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -8,10 +8,7 @@ module Gitlab CACHE_EXPIRE_IN = 1.hour MAX_OFFSET = 2**31 - attr_reader :commit, :project, :path, :offset, :limit, :user - - attr_reader :resolved_commits - private :resolved_commits + attr_reader :commit, :project, :path, :offset, :limit, :user, :resolved_commits def initialize(commit, project, user, params = {}) @commit = commit @@ -34,44 +31,37 @@ module Gitlab # # - An Array of Hashes containing the following keys: # - file_name: The full path of the tree entry - # - type: One of :blob, :tree, or :submodule # - commit: The last ::Commit to touch this entry in the tree # - commit_path: URI of the commit in the web interface - # - An Array of the unique ::Commit objects in the first value + # - commit_title_html: Rendered commit title def summarize - summary = contents - .tap { |summary| fill_last_commits!(summary) } - - [summary, commits] - end - - def fetch_logs - logs, _ = summarize + commits_hsh = fetch_last_cached_commits_list + prerender_commit_full_titles!(commits_hsh.values) - new_offset = next_offset if more? + commits_hsh.map do |path_key, commit| + commit = cache_commit(commit) - [logs.as_json, new_offset] + { + file_name: File.basename(path_key).force_encoding(Encoding::UTF_8), + commit: commit, + commit_path: commit_path(commit), + commit_title_html: markdown_field(commit, :full_title) + } + end end - # Does the tree contain more entries after the given offset + limit? - def more? - all_contents[next_offset].present? - end + def fetch_logs + logs = summarize - # The offset of the next batch of tree entries. If more? returns false, this - # batch will be empty - def next_offset - [all_contents.size + 1, offset + limit].min + [logs.first(limit).as_json, next_offset(logs.size)] end private - def contents - all_contents[offset, limit] || [] - end + def next_offset(entries_count) + return if entries_count <= limit - def commits - resolved_commits.values + offset + limit end def repository @@ -83,32 +73,12 @@ module Gitlab File.join(*[path, ""]) if path end - def entry_path(entry) - File.join(*[path, entry[:file_name]].compact).force_encoding(Encoding::ASCII_8BIT) - end - - def fill_last_commits!(entries) - commits_hsh = fetch_last_cached_commits_list - prerender_commit_full_titles!(commits_hsh.values) - - entries.each do |entry| - path_key = entry_path(entry) - commit = cache_commit(commits_hsh[path_key]) - - if commit - entry[:commit] = commit - entry[:commit_path] = commit_path(commit) - entry[:commit_title_html] = markdown_field(commit, :full_title) - end - end - end - def fetch_last_cached_commits_list - cache_key = ['projects', project.id, 'last_commits', commit.id, ensured_path, offset, limit] + cache_key = ['projects', project.id, 'last_commits', commit.id, ensured_path, offset, limit + 1] commits = Rails.cache.fetch(cache_key, expires_in: CACHE_EXPIRE_IN) do repository - .list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit, literal_pathspec: true) + .list_last_commits_for_tree(commit.id, ensured_path, offset: offset, limit: limit + 1, literal_pathspec: true) .transform_values! { |commit| commit_to_hash(commit) } end @@ -131,26 +101,6 @@ module Gitlab Gitlab::Routing.url_helpers.project_commit_path(project, commit) end - def all_contents - strong_memoize(:all_contents) { cached_contents } - end - - def cached_contents - cache_key = ['projects', project.id, 'content', commit.id, path] - - Rails.cache.fetch(cache_key, expires_in: CACHE_EXPIRE_IN) do - [ - *tree.trees, - *tree.blobs, - *tree.submodules - ].map { |entry| { file_name: entry.name, type: entry.type } } - end - end - - def tree - strong_memoize(:tree) { repository.tree(commit.id, path) } - end - def prerender_commit_full_titles!(commits) # Preload commit authors as they are used in rendering commits.each(&:lazy_author) diff --git a/lib/gitlab/usage/metrics/instrumentations/unique_active_users_metric.rb b/lib/gitlab/usage/metrics/instrumentations/unique_active_users_metric.rb deleted file mode 100644 index 9da30db05dd..00000000000 --- a/lib/gitlab/usage/metrics/instrumentations/unique_active_users_metric.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Usage - module Metrics - module Instrumentations - class UniqueActiveUsersMetric < DatabaseMetric - operation :count - relation { ::User.active } - - metric_options do - { - batch_size: 10_000 - } - end - - def time_constraints - case time_frame - when '28d' - monthly_time_range_db_params(column: :last_activity_on) - else - super - end - end - end - end - end - end -end diff --git a/lib/gitlab/usage/service_ping/instrumented_payload.rb b/lib/gitlab/usage/service_ping/instrumented_payload.rb index 6cc67321ba1..3aa6789a010 100644 --- a/lib/gitlab/usage/service_ping/instrumented_payload.rb +++ b/lib/gitlab/usage/service_ping/instrumented_payload.rb @@ -34,6 +34,13 @@ module Gitlab return {} unless definition.present? Gitlab::Usage::Metric.new(definition).method(output_method).call + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + metric_fallback(key_path) + end + + def metric_fallback(key_path) + ::Gitlab::Usage::Metrics::KeyPathProcessor.process(key_path, ::Gitlab::Utils::UsageData::FALLBACK) end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 604fa364aa2..6f36a09fe48 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -105,7 +105,6 @@ module Gitlab clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), clusters_management_project: count(::Clusters::Cluster.with_management_project), - clusters_integrations_elastic_stack: count(::Clusters::Integrations::ElasticStack.enabled), clusters_integrations_prometheus: count(::Clusters::Integrations::Prometheus.enabled), kubernetes_agents: count(::Clusters::Agent), kubernetes_agents_with_token: distinct_count(::Clusters::AgentToken, :agent_id), @@ -135,7 +134,6 @@ module Gitlab projects_creating_incidents: distinct_count(Issue.incident, :project_id), projects_imported_from_github: count(Project.where(import_type: 'github')), projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), - projects_with_tracing_enabled: count(ProjectTracingSetting), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), @@ -558,7 +556,6 @@ module Gitlab operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period), start: minimum_id(User), finish: maximum_id(User)), - projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id), projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id), projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id), projects_with_alert_incidents: distinct_count(::Issue.incident.with_alert_management_alerts.where(time_period), :project_id), @@ -654,8 +651,6 @@ module Gitlab end def with_duration - return yield unless Feature.enabled?(:measure_service_ping_metric_collection) - result = nil duration = Benchmark.realtime do result = yield diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index 8feb24e49ac..5ede840661a 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -10,24 +10,24 @@ module Gitlab EDIT_BY_LIVE_PREVIEW = 'g_edit_by_live_preview' class << self - def track_web_ide_edit_action(author:, time: Time.zone.now) - track_unique_action(EDIT_BY_WEB_IDE, author, time) + def track_web_ide_edit_action(author:, time: Time.zone.now, project:) + track_unique_action(EDIT_BY_WEB_IDE, author, time, project) end def count_web_ide_edit_actions(date_from:, date_to:) count_unique(EDIT_BY_WEB_IDE, date_from, date_to) end - def track_sfe_edit_action(author:, time: Time.zone.now) - track_unique_action(EDIT_BY_SFE, author, time) + def track_sfe_edit_action(author:, time: Time.zone.now, project:) + track_unique_action(EDIT_BY_SFE, author, time, project) end def count_sfe_edit_actions(date_from:, date_to:) count_unique(EDIT_BY_SFE, date_from, date_to) end - def track_snippet_editor_edit_action(author:, time: Time.zone.now) - track_unique_action(EDIT_BY_SNIPPET_EDITOR, author, time) + def track_snippet_editor_edit_action(author:, time: Time.zone.now, project:) + track_unique_action(EDIT_BY_SNIPPET_EDITOR, author, time, project) end def count_snippet_editor_edit_actions(date_from:, date_to:) @@ -39,15 +39,25 @@ module Gitlab count_unique(events, date_from, date_to) end - def track_live_preview_edit_action(author:, time: Time.zone.now) - track_unique_action(EDIT_BY_LIVE_PREVIEW, author, time) + def track_live_preview_edit_action(author:, time: Time.zone.now, project:) + track_unique_action(EDIT_BY_LIVE_PREVIEW, author, time, project) end private - def track_unique_action(action, author, time) + def track_unique_action(action, author, time, project = nil) return unless author + if Feature.enabled?(:route_hll_to_snowplow_phase2) + Gitlab::Tracking.event( + 'ide_edit', + action.to_s, + project: project, + namespace: project&.namespace, + user: author + ) + end + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 0ace6e99c59..40581bda81b 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -33,10 +33,24 @@ module Gitlab pipeline_authoring quickactions search - testing user_packages ].freeze + CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS = %w[ + ci_users + error_tracking + ide_edit + importer + incident_management_alerts + pipeline_authoring + secure + snippets + source_code + terraform + testing + work_items + ].freeze + # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id # @@ -114,7 +128,7 @@ module Gitlab # - Most of the metrics have weekly aggregation. We recommend this as it generates fewer keys in Redis to store. # - The aggregation used doesn't affect data granulation. def unique_events_data - categories.each_with_object({}) do |category, category_results| + categories_pending_migration.each_with_object({}) do |category, category_results| events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| @@ -148,6 +162,14 @@ module Gitlab private + def categories_pending_migration + if ::Feature.enabled?(:use_redis_hll_instrumentation_classes) + (categories - CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS) + else + categories + end + end + def track(values, event_name, context: '', time: Time.zone.now) return unless ::ServicePing::ServicePingSettings.enabled? diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 083de402175..9d463e11772 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -149,6 +149,19 @@ module Gitlab Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id) end + + def track_snowplow_action(action, author, project) + return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project&.namespace) + return unless author + + Gitlab::Tracking.event( + ISSUE_CATEGORY, + action.to_s, + project: project, + namespace: project&.namespace, + user: author + ) + end end end end diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml index 5a1e7f03278..76c97a974d7 100644 --- a/lib/gitlab/usage_data_counters/known_events/analytics.yml +++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml @@ -78,3 +78,31 @@ category: analytics redis_slot: analytics aggregation: weekly +- name: p_analytics_ci_cd_time_to_restore_service + category: analytics + redis_slot: analytics + aggregation: weekly +- name: p_analytics_ci_cd_change_failure_rate + category: analytics + redis_slot: analytics + aggregation: weekly +- name: g_analytics_ci_cd_release_statistics + category: analytics + redis_slot: analytics + aggregation: weekly +- name: g_analytics_ci_cd_deployment_frequency + category: analytics + redis_slot: analytics + aggregation: weekly +- name: g_analytics_ci_cd_lead_time + category: analytics + redis_slot: analytics + aggregation: weekly +- name: g_analytics_ci_cd_time_to_restore_service + category: analytics + redis_slot: analytics + aggregation: weekly +- name: g_analytics_ci_cd_change_failure_rate + category: analytics + redis_slot: analytics + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_users.yml b/lib/gitlab/usage_data_counters/known_events/ci_users.yml index 5159dcf62ab..b012d61eef5 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_users.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_users.yml @@ -3,3 +3,8 @@ redis_slot: ci_users aggregation: weekly feature_flag: +- name: ci_users_executing_verify_environment_job + category: ci_users + redis_slot: ci_users + aggregation: weekly + feature_flag: diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 0dcbaf59c9c..88c9f44c165 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -157,34 +157,6 @@ category: testing redis_slot: testing aggregation: weekly -- name: i_testing_metrics_report_widget_total - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_group_code_coverage_visit_total - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_full_code_quality_report_total - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_web_performance_widget_total - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_group_code_coverage_project_click_total - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_load_performance_widget_total - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_metrics_report_artifact_uploaders - category: testing - redis_slot: testing - aggregation: weekly - name: i_testing_summary_widget_total category: testing redis_slot: testing @@ -390,3 +362,8 @@ category: growth redis_slot: users aggregation: weekly +# Manage +- name: unique_active_user + category: manage + aggregation: weekly + expiry: 42 diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 9c0f8fe9a80..fbb03a31a6f 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -76,8 +76,19 @@ module Gitlab track_unique_action_by_user(MR_REOPEN_ACTION, user) end - def track_approve_mr_action(user:) + def track_approve_mr_action(user:, merge_request:) track_unique_action_by_user(MR_APPROVE_ACTION, user) + + project = merge_request.target_project + return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) + + Gitlab::Tracking.event( + 'merge_requests', + MR_APPROVE_ACTION, + project: project, + namespace: project.namespace, + user: user + ) end def track_unapprove_mr_action(user:) diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb index 6f5300405c7..51bca8b51fe 100644 --- a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb @@ -26,3 +26,7 @@ module Gitlab end end end + +# rubocop:disable Layout/LineLength +Gitlab::UsageDataCounters::WorkItemActivityUniqueCounter.prepend_mod_with('Gitlab::UsageDataCounters::WorkItemActivityUniqueCounter') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/version_info.rb b/lib/gitlab/version_info.rb index aa6d5310161..f967a12b959 100644 --- a/lib/gitlab/version_info.rb +++ b/lib/gitlab/version_info.rb @@ -6,20 +6,27 @@ module Gitlab attr_reader :major, :minor, :patch - def self.parse(str) - if str && m = str.match(/(\d+)\.(\d+)\.(\d+)/) - VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i) + VERSION_REGEX = /(\d+)\.(\d+)\.(\d+)/.freeze + + def self.parse(str, parse_suffix: false) + if str.is_a?(self.class) + str + elsif str && m = str.match(VERSION_REGEX) + VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i, parse_suffix ? m.post_match : nil) else VersionInfo.new end end - def initialize(major = 0, minor = 0, patch = 0) + def initialize(major = 0, minor = 0, patch = 0, suffix = nil) @major = major @minor = minor @patch = patch + @suffix_s = suffix.to_s end + # rubocop:disable Metrics/CyclomaticComplexity + # rubocop:disable Metrics/PerceivedComplexity def <=>(other) return unless other.is_a? VersionInfo return unless valid? && other.valid? @@ -36,21 +43,49 @@ module Gitlab 1 elsif @patch < other.patch -1 + elsif @suffix_s.empty? && other.suffix.present? + 1 + elsif other.suffix.empty? && @suffix_s.present? + -1 else - 0 + suffix <=> other.suffix end end + # rubocop:enable Metrics/CyclomaticComplexity + # rubocop:enable Metrics/PerceivedComplexity def to_s if valid? - "%d.%d.%d" % [@major, @minor, @patch] + "%d.%d.%d%s" % [@major, @minor, @patch, @suffix_s] else - "Unknown" + 'Unknown' end end + def suffix + @suffix ||= @suffix_s.strip.gsub('-', '.pre.').scan(/\d+|[a-z]+/i).map do |s| + /^\d+$/ =~ s ? s.to_i : s + end.freeze + end + def valid? @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 end + + def hash + [self.class, to_s].hash + end + + def eql?(other) + (self <=> other) == 0 + end + + def same_minor_version?(other) + @major == other.major && @minor == other.minor + end + + def without_patch + self.class.new(@major, @minor, 0) + end end end diff --git a/lib/gitlab/wiki_pages/front_matter_parser.rb b/lib/gitlab/wiki_pages/front_matter_parser.rb index ee30fa907f4..071b0dde619 100644 --- a/lib/gitlab/wiki_pages/front_matter_parser.rb +++ b/lib/gitlab/wiki_pages/front_matter_parser.rb @@ -3,6 +3,8 @@ module Gitlab module WikiPages class FrontMatterParser + FEATURE_FLAG = :wiki_front_matter + # We limit the maximum length of text we are prepared to parse as YAML, to # avoid exploitations and attempts to consume memory and CPU. We allow for: # - a title line @@ -28,12 +30,18 @@ module Gitlab end # @param [String] wiki_content - def initialize(wiki_content) + # @param [FeatureGate] feature_gate The scope for feature availability + def initialize(wiki_content, feature_gate) @wiki_content = wiki_content + @feature_gate = feature_gate + end + + def self.enabled?(gate = nil) + Feature.enabled?(FEATURE_FLAG, gate) end def parse - return empty_result unless wiki_content.present? + return empty_result unless enabled? && wiki_content.present? return empty_result(block.error) unless block.valid? Result.new(front_matter: block.data, content: strip_front_matter_block) @@ -86,12 +94,16 @@ module Gitlab private - attr_reader :wiki_content + attr_reader :wiki_content, :feature_gate def empty_result(reason = nil, error = nil) Result.new(content: wiki_content, reason: reason, error: error) end + def enabled? + self.class.enabled?(feature_gate) + end + def block @block ||= parse_front_matter_block end diff --git a/lib/gitlab/x509/certificate.rb b/lib/gitlab/x509/certificate.rb index 752f3c6b004..98688f504eb 100644 --- a/lib/gitlab/x509/certificate.rb +++ b/lib/gitlab/x509/certificate.rb @@ -23,6 +23,18 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize end + def self.default_cert_dir + strong_memoize(:default_cert_dir) do + ENV.fetch('SSL_CERT_DIR', OpenSSL::X509::DEFAULT_CERT_DIR) + end + end + + def self.default_cert_file + strong_memoize(:default_cert_file) do + ENV.fetch('SSL_CERT_FILE', OpenSSL::X509::DEFAULT_CERT_FILE) + end + end + def self.from_strings(key_string, cert_string, ca_certs_string = nil) key = OpenSSL::PKey::RSA.new(key_string) cert = OpenSSL::X509::Certificate.new(cert_string) @@ -39,10 +51,10 @@ module Gitlab # Returns all top-level, readable files in the default CA cert directory def self.ca_certs_paths - cert_paths = Dir["#{OpenSSL::X509::DEFAULT_CERT_DIR}/*"].select do |path| + cert_paths = Dir["#{default_cert_dir}/*"].select do |path| !File.directory?(path) && File.readable?(path) end - cert_paths << OpenSSL::X509::DEFAULT_CERT_FILE if File.exist? OpenSSL::X509::DEFAULT_CERT_FILE + cert_paths << default_cert_file if File.exist? default_cert_file cert_paths end @@ -61,6 +73,11 @@ module Gitlab clear_memoization(:ca_certs_bundle) end + def self.reset_default_cert_paths + clear_memoization(:default_cert_dir) + clear_memoization(:default_cert_file) + end + # Returns an array of OpenSSL::X509::Certificate objects, empty array if none found # # Ruby OpenSSL::X509::Certificate.new will only load the first diff --git a/lib/gitlab/x509/commit.rb b/lib/gitlab/x509/commit.rb index c7f4b7cbdf5..3636e776a44 100644 --- a/lib/gitlab/x509/commit.rb +++ b/lib/gitlab/x509/commit.rb @@ -5,30 +5,10 @@ require 'digest' module Gitlab module X509 class Commit < Gitlab::SignedCommit - def signature - super - - return @signature if @signature - - cached_signature = lazy_signature&.itself - return @signature = cached_signature if cached_signature.present? - - @signature = create_cached_signature! - end - - def update_signature!(cached_signature) - cached_signature.update!(attributes) - @signature = cached_signature - end - private - def lazy_signature - BatchLoader.for(@commit.sha).batch do |shas, loader| - CommitSignatures::X509CommitSignature.by_commit_sha(shas).each do |signature| - loader.call(signature.commit_sha, signature) - end - end + def signature_class + CommitSignatures::X509CommitSignature end def attributes @@ -45,14 +25,6 @@ module Gitlab verification_status: signature.verification_status } end - - def create_cached_signature! - return if attributes.nil? - - return CommitSignatures::X509CommitSignature.new(attributes) if Gitlab::Database.read_only? - - CommitSignatures::X509CommitSignature.safe_create!(attributes) - end end end end diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index a6761e211fa..8acbfc144e9 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -59,7 +59,7 @@ module Gitlab if Feature.enabled?(:x509_forced_cert_loading, type: :ops) # Forcibly load the default cert file because the OpenSSL library seemingly ignores it - store.add_file(OpenSSL::X509::DEFAULT_CERT_FILE) if File.exist?(OpenSSL::X509::DEFAULT_CERT_FILE) + store.add_file(Gitlab::X509::Certificate.default_cert_file) if File.exist?(Gitlab::X509::Certificate.default_cert_file) # rubocop:disable Layout/LineLength end # valid_signing_time? checks the time attributes already |