diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-18 23:02:30 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-18 23:02:30 +0300 |
| commit | 41fe97390ceddf945f3d967b8fdb3de4c66b7dea (patch) | |
| tree | 9c8d89a8624828992f06d892cd2f43818ff5dcc8 /lib/gitlab | |
| parent | 0804d2dc31052fb45a1efecedc8e06ce9bc32862 (diff) | |
Add latest changes from gitlab-org/gitlab@14-9-stable-eev14.9.0-rc42
Diffstat (limited to 'lib/gitlab')
238 files changed, 3398 insertions, 778 deletions
diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index bc9d94ef09c..af695c5cfa4 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -80,12 +80,13 @@ module Gitlab direction: direction&.to_sym, page: page, end_event_filter: end_event_filter.to_sym, - use_aggregated_data_collector: Feature.enabled?(:use_vsa_aggregated_tables, group || project, default_enabled: :yaml) + use_aggregated_data_collector: use_aggregated_backend? }.merge(attributes.symbolize_keys.slice(*FINDER_PARAM_NAMES)) end def to_data_attributes {}.tap do |attrs| + attrs[:aggregation] = aggregation_attributes if group attrs[:group] = group_data_attributes if group attrs[:value_stream] = value_stream_data_attributes.to_json if value_stream attrs[:created_after] = created_after.to_date.iso8601 @@ -103,6 +104,24 @@ module Gitlab private + def use_aggregated_backend? + group.present? && # for now it's only available on the group-level + aggregation.enabled && + Feature.enabled?(:use_vsa_aggregated_tables, group, default_enabled: :yaml) + end + + def aggregation_attributes + { + enabled: aggregation.enabled.to_s, + last_run_at: aggregation.last_incremental_run_at&.iso8601, + next_run_at: aggregation.estimated_next_run_at&.iso8601 + } + end + + def aggregation + @aggregation ||= ::Analytics::CycleAnalytics::Aggregation.safe_create_for_group(group) + end + def group_data_attributes { id: group.id, diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index d2a31938e89..0b0aaacbaff 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -39,7 +39,8 @@ module Gitlab profile_update_username: { threshold: 10, interval: 1.minute }, update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, - user_email_lookup: { threshold: -> { application_settings.user_email_lookup_limit }, interval: 1.minute }, + search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute }, + search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, gitlab_shell_operation: { threshold: 600, interval: 1.minute } }.freeze end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index ecda96af403..7adaaef86e4 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -12,6 +12,7 @@ module Gitlab class InsufficientScopeError < AuthenticationError attr_reader :scopes + def initialize(scopes) @scopes = scopes.map { |s| s.try(:name) || s } end diff --git a/lib/gitlab/auth/ldap/user.rb b/lib/gitlab/auth/ldap/user.rb index d134350775d..56c2af1910e 100644 --- a/lib/gitlab/auth/ldap/user.rb +++ b/lib/gitlab/auth/ldap/user.rb @@ -11,9 +11,6 @@ module Gitlab module Ldap class User < Gitlab::Auth::OAuth::User extend ::Gitlab::Utils::Override - def save - super('LDAP') - end # instance methods def find_user @@ -44,6 +41,10 @@ module Gitlab def auth_hash=(auth_hash) @auth_hash = Gitlab::Auth::Ldap::AuthHash.new(auth_hash) end + + def protocol_name + 'LDAP' + end end end end diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 2ec75669d24..a45778159c7 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -7,6 +7,7 @@ module Gitlab module OAuth class AuthHash attr_reader :auth_hash + def initialize(auth_hash) @auth_hash = auth_hash end diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 9f142727ebb..200f1a843e6 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -46,7 +46,7 @@ module Gitlab valid? && persisted? end - def save(provider = 'OAuth') + def save(provider = protocol_name) raise SigninDisabledForProviderError if oauth_provider_disabled? raise SignupDisabledError unless gl_user @@ -55,6 +55,7 @@ module Gitlab Users::UpdateService.new(gl_user, user: gl_user).execute! gl_user.block_pending_approval if block_after_save + activate_user_if_user_cap_not_reached log.info "(#{provider}) saving user #{auth_hash.email} from login with admin => #{gl_user.admin}, extern_uid => #{auth_hash.uid}" gl_user @@ -96,8 +97,16 @@ module Gitlab end end + def protocol_name + 'OAuth' + end + protected + def activate_user_if_user_cap_not_reached + nil + end + def should_save? true end diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index b6ed6bbf2df..0948663b4b3 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -42,6 +42,10 @@ module Gitlab nil end + def can_sign_in_bot?(user) + user&.project_bot? && api_request? + end + # To prevent Rack Attack from incorrectly rate limiting # authenticated Git activity, we need to authenticate the user # from other means (e.g. HTTP Basic Authentication) only if the diff --git a/lib/gitlab/auth/saml/user.rb b/lib/gitlab/auth/saml/user.rb index 205d5fe0015..d14da41deb6 100644 --- a/lib/gitlab/auth/saml/user.rb +++ b/lib/gitlab/auth/saml/user.rb @@ -11,10 +11,6 @@ module Gitlab class User < Gitlab::Auth::OAuth::User extend ::Gitlab::Utils::Override - def save - super('SAML') - end - def find_user user = find_by_uid_and_provider @@ -40,6 +36,10 @@ module Gitlab saml_config.upstream_two_factor_authn_contexts&.include?(auth_hash.authn_context) end + def protocol_name + 'SAML' + end + protected def saml_config diff --git a/lib/gitlab/background_migration/backfill_issue_search_data.rb b/lib/gitlab/background_migration/backfill_issue_search_data.rb new file mode 100644 index 00000000000..ec206cbfd41 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_issue_search_data.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true +# rubocop:disable Style/Documentation + +module Gitlab + module BackgroundMigration + # Backfills the new `issue_search_data` table, which contains + # the tsvector from the issue title and description. + class BackfillIssueSearchData + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, stop_id, batch_table, batch_column, sub_batch_size, pause_ms) + define_batchable_model(batch_table, connection: ActiveRecord::Base.connection).where(batch_column => start_id..stop_id).each_batch(of: sub_batch_size) do |sub_batch| + update_search_data(sub_batch) + + sleep(pause_ms * 0.001) + rescue ActiveRecord::StatementInvalid => e + raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') + + update_search_data_individually(sub_batch, pause_ms) + end + end + + private + + def update_search_data(relation) + relation.klass.connection.execute( + <<~SQL + INSERT INTO issue_search_data (project_id, issue_id, search_vector, created_at, updated_at) + SELECT + project_id, + id, + setweight(to_tsvector('english', LEFT(title, 255)), 'A') || setweight(to_tsvector('english', LEFT(REGEXP_REPLACE(description, '[A-Za-z0-9+/@]{50,}', ' ', 'g'), 1048576)), 'B'), + NOW(), + NOW() + FROM issues + WHERE issues.id IN (#{relation.select(:id).to_sql}) + ON CONFLICT DO NOTHING + SQL + ) + end + + def update_search_data_individually(relation, pause_ms) + relation.pluck(:id).each do |issue_id| + update_search_data(relation.klass.where(id: issue_id)) + + sleep(pause_ms * 0.001) + rescue ActiveRecord::StatementInvalid => e + raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') + + logger.error( + message: 'Error updating search data: string is too long for tsvector', + class: relation.klass.name, + model_id: issue_id + ) + end + end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb index 61145f6a445..669e5338dd1 100644 --- a/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb +++ b/lib/gitlab/background_migration/backfill_jira_tracker_deployment_type2.rb @@ -79,7 +79,7 @@ module Gitlab end def mark_jobs_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name.demodulize, arguments) end end end diff --git a/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb b/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb new file mode 100644 index 00000000000..1ed147d67c7 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_member_namespace_for_group_members.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the `members.member_namespace_id` column for `type=GroupMember` + class BackfillMemberNamespaceForGroupMembers + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + batch_metrics.time_operation(:update_all) do + sub_batch.update_all('member_namespace_id=source_id') + end + + pause_ms = [0, pause_ms].max + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table, connection: ActiveRecord::Base.connection) + .joins('INNER JOIN namespaces ON members.source_id = namespaces.id') + .where(source_key_column => start_id..stop_id) + .where(type: 'GroupMember') + .where(source_type: 'Namespace') + .where(member_namespace_id: nil) + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/base_strategy.rb b/lib/gitlab/background_migration/batching_strategies/base_strategy.rb new file mode 100644 index 00000000000..37bddea4f61 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/base_strategy.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Simple base class for batching strategy job classes. + # + # Any strategy class that inherits from the base class will have connection to the tracking database set on + # initialization. + class BaseStrategy + def initialize(connection:) + @connection = connection + end + + def next_batch(*arguments) + raise NotImplementedError, + "#{self.class} does not implement #{__method__}" + end + + private + + attr_reader :connection + end + end + end +end diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb index 09700438d47..5569bac0e19 100644 --- a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb @@ -8,7 +8,7 @@ module Gitlab # values for the next batch as an array. # # If no more batches exist in the table, returns nil. - class PrimaryKeyBatchingStrategy + class PrimaryKeyBatchingStrategy < BaseStrategy include Gitlab::Database::DynamicModelHelpers # Finds and returns the next batch in the table. @@ -19,7 +19,7 @@ module Gitlab # batch_size - The size of the next batch # job_arguments - The migration job arguments def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:) - model_class = define_batchable_model(table_name, connection: ActiveRecord::Base.connection) + model_class = define_batchable_model(table_name, connection: connection) quoted_column_name = model_class.connection.quote_column_name(column_name) relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb new file mode 100644 index 00000000000..3843356af69 --- /dev/null +++ b/lib/gitlab/background_migration/encrypt_integration_properties.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates the integration.properties column from plaintext to encrypted text. + class EncryptIntegrationProperties + # The Integration model, with just the relevant bits. + class Integration < ActiveRecord::Base + include EachBatch + + ALGORITHM = 'aes-256-gcm' + + self.table_name = 'integrations' + self.inheritance_column = :_type_disabled + + scope :with_properties, -> { where.not(properties: nil) } + scope :not_already_encrypted, -> { where(encrypted_properties: nil) } + scope :for_batch, ->(range) { where(id: range) } + + attr_encrypted :encrypted_properties_tmp, + attribute: :encrypted_properties, + mode: :per_attribute_iv, + key: ::Settings.attr_encrypted_db_key_base_32, + algorithm: ALGORITHM, + marshal: true, + marshaler: ::Gitlab::Json, + encode: false, + encode_iv: false + + # See 'Integration#reencrypt_properties' + def encrypt_properties + data = ::Gitlab::Json.parse(properties) + iv = generate_iv(ALGORITHM) + ep = self.class.encrypt(:encrypted_properties_tmp, data, { iv: iv }) + + [ep, iv] + end + end + + def perform(start_id, stop_id) + batch_query = Integration.with_properties.not_already_encrypted.for_batch(start_id..stop_id) + encrypt_batch(batch_query) + mark_job_as_succeeded(start_id, stop_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + self.class.name.demodulize, + arguments + ) + end + + # represent binary string as a PSQL binary literal: + # https://www.postgresql.org/docs/9.4/datatype-binary.html + def bytea(value) + "'\\x#{value.unpack1('H*')}'::bytea" + end + + def encrypt_batch(batch_query) + values = batch_query.select(:id, :properties).map do |record| + encrypted_properties, encrypted_properties_iv = record.encrypt_properties + "(#{record.id}, #{bytea(encrypted_properties)}, #{bytea(encrypted_properties_iv)})" + end + + return if values.empty? + + Integration.connection.execute(<<~SQL.squish) + WITH cte(cte_id, cte_encrypted_properties, cte_encrypted_properties_iv) + AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + SELECT * + FROM (VALUES #{values.join(',')}) AS t (id, encrypted_properties, encrypted_properties_iv) + ) + UPDATE #{Integration.table_name} + SET encrypted_properties = cte_encrypted_properties + , encrypted_properties_iv = cte_encrypted_properties_iv + FROM cte + WHERE cte_id = id + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb index 2b049ea2d2f..a34e923545c 100644 --- a/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb +++ b/lib/gitlab/background_migration/fix_vulnerability_occurrences_with_hashes_as_raw_metadata.rb @@ -59,7 +59,7 @@ module Gitlab private def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( 'FixVulnerabilityOccurrencesWithHashesAsRawMetadata', arguments ) diff --git a/lib/gitlab/background_migration/job_coordinator.rb b/lib/gitlab/background_migration/job_coordinator.rb index b7d47c389df..acbb5f76ad8 100644 --- a/lib/gitlab/background_migration/job_coordinator.rb +++ b/lib/gitlab/background_migration/job_coordinator.rb @@ -50,34 +50,41 @@ module Gitlab Gitlab::Database::SharedModel.using_connection(connection, &block) end - def steal(steal_class, retry_dead_jobs: false) - with_shared_connection do + def pending_jobs(include_dead_jobs: false) + Enumerator.new do |y| queues = [ Sidekiq::ScheduledSet.new, Sidekiq::Queue.new(self.queue) ] - if retry_dead_jobs + if include_dead_jobs queues << Sidekiq::RetrySet.new queues << Sidekiq::DeadSet.new end queues.each do |queue| queue.each do |job| - migration_class, migration_args = job.args + y << job if job.klass == worker_class.name + end + end + end + end + + def steal(steal_class, retry_dead_jobs: false) + with_shared_connection do + pending_jobs(include_dead_jobs: retry_dead_jobs).each do |job| + migration_class, migration_args = job.args - next unless job.klass == worker_class.name - next unless migration_class == steal_class - next if block_given? && !(yield job) + next unless migration_class == steal_class + next if block_given? && !(yield job) - begin - perform(migration_class, migration_args) if job.delete - rescue Exception # rubocop:disable Lint/RescueException - worker_class # enqueue this migration again - .perform_async(migration_class, migration_args) + begin + perform(migration_class, migration_args) if job.delete + rescue Exception # rubocop:disable Lint/RescueException + worker_class # enqueue this migration again + .perform_async(migration_class, migration_args) - raise - end + raise end end end diff --git a/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb b/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb new file mode 100644 index 00000000000..49eff6e2771 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_personal_namespace_project_maintainer_to_owner.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates personal namespace project `maintainer` memberships (for the associated user only) to OWNER + # Does not create any missing records, simply migrates existing ones + class MigratePersonalNamespaceProjectMaintainerToOwner + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + batch_metrics.time_operation(:update_all) do + sub_batch.update_all('access_level = 50') + end + + pause_ms = 0 if pause_ms < 0 + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + # members of projects within their own personal namespace + + # rubocop: disable CodeReuse/ActiveRecord + define_batchable_model(:members, connection: ApplicationRecord.connection) + .where(source_key_column => start_id..stop_id) + .joins("INNER JOIN projects ON members.source_id = projects.id") + .joins("INNER JOIN namespaces ON projects.namespace_id = namespaces.id") + .where(type: 'ProjectMember') + .where("namespaces.type = 'User'") + .where('members.access_level < 50') + .where('namespaces.owner_id = members.user_id') + end + end + # rubocop: enable CodeReuse/ActiveRecord + end +end diff --git a/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb new file mode 100644 index 00000000000..78e897d9ae1 --- /dev/null +++ b/lib/gitlab/background_migration/nullify_orphan_runner_id_on_ci_builds.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify orphan runner_id on ci_builds table + class NullifyOrphanRunnerIdOnCiBuilds + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) + pause_ms = 0 if pause_ms < 0 + + batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch| + batch_metrics.time_operation(:update_all) do + sub_batch.update_all(runner_id: nil) + end + + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def connection + ActiveRecord::Base.connection + end + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table, connection: connection) + .joins('LEFT OUTER JOIN ci_runners ON ci_runners.id = ci_builds.runner_id') + .where('ci_builds.runner_id IS NOT NULL AND ci_runners.id IS NULL') + .where(source_key_column => start_id..stop_id) + end + end + end +end diff --git a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb index ba3f7c47047..c34cc57ce60 100644 --- a/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb +++ b/lib/gitlab/background_migration/project_namespaces/backfill_project_namespaces.rb @@ -34,8 +34,11 @@ module Gitlab def backfill_project_namespaces(namespace_id) project_ids.each_slice(sub_batch_size) do |project_ids| - ActiveRecord::Base.connection.execute("select gin_clean_pending_list('index_namespaces_on_name_trigram')") - ActiveRecord::Base.connection.execute("select gin_clean_pending_list('index_namespaces_on_path_trigram')") + # cleanup gin indexes on namespaces table + cleanup_gin_index('namespaces') + + # cleanup gin indexes on projects table + cleanup_gin_index('projects') # We need to lock these project records for the period when we create project namespaces # and link them to projects so that if a project is modified in the time between creating @@ -53,6 +56,14 @@ module Gitlab end end + def cleanup_gin_index(table_name) + index_names = ActiveRecord::Base.connection.select_values("select indexname::text from pg_indexes where tablename = '#{table_name}' and indexdef ilike '%gin%'") + + index_names.each do |index_name| + ActiveRecord::Base.connection.execute("select gin_clean_pending_list('#{index_name}')") + end + end + def cleanup_backfilled_project_namespaces(namespace_id) project_ids.each_slice(sub_batch_size) do |project_ids| # IMPORTANT: first nullify project_namespace_id in projects table to avoid removing projects when records diff --git a/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb b/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb new file mode 100644 index 00000000000..d47aa76f24b --- /dev/null +++ b/lib/gitlab/background_migration/remove_all_trace_expiration_dates.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Removing expire_at timestamps that shouldn't have + # been written to traces on gitlab.com. + class RemoveAllTraceExpirationDates + include Gitlab::Database::MigrationHelpers + + BATCH_SIZE = 1_000 + + # Stubbed class to connect to the CI database + # connects_to has to be called in abstract classes. + class MultiDbAdaptableClass < ActiveRecord::Base + self.abstract_class = true + + if Gitlab::Database.has_config?(:ci) + connects_to database: { writing: :ci, reading: :ci } + end + end + + # Stubbed class to access the ci_job_artifacts table + class JobArtifact < MultiDbAdaptableClass + include EachBatch + + self.table_name = 'ci_job_artifacts' + + TARGET_TIMESTAMPS = [ + Date.new(2021, 04, 22).midnight.utc, + Date.new(2021, 05, 22).midnight.utc, + Date.new(2021, 06, 22).midnight.utc, + Date.new(2022, 01, 22).midnight.utc, + Date.new(2022, 02, 22).midnight.utc, + Date.new(2022, 03, 22).midnight.utc, + Date.new(2022, 04, 22).midnight.utc + ].freeze + + scope :traces, -> { where(file_type: 3) } + scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } + scope :in_targeted_timestamps, -> { where(expire_at: TARGET_TIMESTAMPS) } + end + + def perform(start_id, end_id) + return unless Gitlab.com? + + JobArtifact.traces + .between(start_id, end_id) + .in_targeted_timestamps + .each_batch(of: BATCH_SIZE) { |batch| batch.update_all(expire_at: nil) } + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb new file mode 100644 index 00000000000..80ca76ef37f --- /dev/null +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values_on_projects.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify duplicate runners_token_encrypted values in projects table in batches + class ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects + class Project < ActiveRecord::Base # rubocop:disable Style/Documentation + include ::EachBatch + + self.table_name = 'projects' + + scope :base_query, -> do + where.not(runners_token_encrypted: nil) + end + end + + def perform(start_id, end_id) + # Reset duplicate runner tokens that would prevent creating an unique index. + duplicate_tokens = Project.base_query + .where(id: start_id..end_id) + .group(:runners_token_encrypted) + .having('COUNT(*) > 1') + .pluck(:runners_token_encrypted) + + Project.where(runners_token_encrypted: duplicate_tokens).update_all(runners_token_encrypted: nil) if duplicate_tokens.any? + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('ResetDuplicateCiRunnersTokenEncryptedValuesOnProjects', arguments) + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb new file mode 100644 index 00000000000..d87ce6c88d3 --- /dev/null +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values_on_projects.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to nullify duplicate ci_runners_token values in projects table in batches + class ResetDuplicateCiRunnersTokenValuesOnProjects + class Project < ActiveRecord::Base # rubocop:disable Style/Documentation + include ::EachBatch + + self.table_name = 'projects' + + scope :base_query, -> do + where.not(runners_token: nil) + end + end + + def perform(start_id, end_id) + # Reset duplicate runner tokens that would prevent creating an unique index. + duplicate_tokens = Project.base_query + .where(id: start_id..end_id) + .group(:runners_token) + .having('COUNT(*) > 1') + .pluck(:runners_token) + + Project.where(runners_token: duplicate_tokens).update_all(runners_token: nil) if duplicate_tokens.any? + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('ResetDuplicateCiRunnerValuesTokensOnProjects', arguments) + end + end + end +end diff --git a/lib/gitlab/checks/base_bulk_checker.rb b/lib/gitlab/checks/base_bulk_checker.rb index 46a68fdf485..e2a016a9907 100644 --- a/lib/gitlab/checks/base_bulk_checker.rb +++ b/lib/gitlab/checks/base_bulk_checker.rb @@ -4,6 +4,7 @@ module Gitlab module Checks class BaseBulkChecker < BaseChecker attr_reader :changes_access + delegate(*ChangesAccess::ATTRIBUTES, to: :changes_access) def initialize(changes_access) diff --git a/lib/gitlab/checks/base_single_checker.rb b/lib/gitlab/checks/base_single_checker.rb index 06519833d7c..435f4ccf5ba 100644 --- a/lib/gitlab/checks/base_single_checker.rb +++ b/lib/gitlab/checks/base_single_checker.rb @@ -4,6 +4,7 @@ module Gitlab module Checks class BaseSingleChecker < BaseChecker attr_reader :change_access + delegate(*SingleChangeAccess::ATTRIBUTES, to: :change_access) def initialize(change_access) diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 7ade9ca5085..2e5f6611e73 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -36,7 +36,7 @@ module Gitlab # the pattern matching does not work for merge requests pipelines if pipeline.branch? || pipeline.tag? regexp = Gitlab::UntrustedRegexp::RubySyntax - .fabricate(pattern, fallback: true, project: pipeline.project) + .fabricate(pattern, project: pipeline.project) if regexp regexp.match?(pipeline.ref) diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 8dd1f686132..06c81fd65dd 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -37,10 +37,12 @@ module Gitlab next unless dependencies.present? next unless needs_value.present? - missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord (Array#pluck) + if needs_value[:job].nil? && needs_value[:cross_dependency].present? + errors.add(:needs, "corresponding to dependencies must be from the same pipeline") + else + missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord (Array#pluck) - if missing_needs.any? - errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") + errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") if missing_needs.any? end end end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 7b14218d3ea..adc3660d950 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -17,7 +17,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable validations do - validates :config, array_of_strings_or_regexps_with_fallback: true + validates :config, array_of_strings_or_regexps: true end def value @@ -38,7 +38,7 @@ module Gitlab validate :variables_expressions_syntax with_options allow_nil: true do - validates :refs, array_of_strings_or_regexps_with_fallback: true + validates :refs, array_of_strings_or_regexps: true validates :kubernetes, allowed_values: %w[active] validates :variables, array_of_strings: true validates :changes, array_of_strings: true diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index e45dbfa243f..f8fce1abc06 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -8,6 +8,7 @@ module Gitlab # Entry that represents a configuration of job artifacts. # class Reports < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable @@ -15,10 +16,13 @@ module Gitlab %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance browser_performance load_performance license_scanning metrics lsif dotenv cobertura terraform accessibility cluster_applications - requirements coverage_fuzzing api_fuzzing cluster_image_scanning].freeze + requirements coverage_fuzzing api_fuzzing cluster_image_scanning + coverage_report].freeze attributes ALLOWED_KEYS + entry :coverage_report, Reports::CoverageReport, description: 'Coverage report configuration.' + validations do validates :config, type: Hash validates :config, allowed_keys: ALLOWED_KEYS @@ -47,10 +51,18 @@ module Gitlab validates :cluster_applications, array_of_strings_or_string: true # DEPRECATED: https://gitlab.com/gitlab-org/gitlab/-/issues/333441 validates :requirements, array_of_strings_or_string: true end + + validates :config, mutually_exclusive_keys: [:coverage_report, :cobertura] end def value - @config.transform_values { |v| Array(v) } + @config.transform_values do |value| + if value.is_a?(Hash) + value + else + Array(value) + end + end end end end diff --git a/lib/gitlab/ci/config/entry/reports/coverage_report.rb b/lib/gitlab/ci/config/entry/reports/coverage_report.rb new file mode 100644 index 00000000000..98119c7fd53 --- /dev/null +++ b/lib/gitlab/ci/config/entry/reports/coverage_report.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Reports + class CoverageReport < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[coverage_format path].freeze + SUPPORTED_COVERAGE = %w[cobertura].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash + validates :config, allowed_keys: ALLOWED_KEYS + + with_options(presence: true) do + validates :coverage_format, inclusion: { in: SUPPORTED_COVERAGE, message: "must be one of supported formats: #{SUPPORTED_COVERAGE.join(', ')}." } + validates :path, type: String + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb index 840f2d6f31a..4722f2e9a61 100644 --- a/lib/gitlab/ci/config/entry/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -24,7 +24,7 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS validates :config, disallowed_keys: %i[start_in], unless: :specifies_delay? validates :start_in, presence: true, if: :specifies_delay? - validates :start_in, duration: { limit: '1 day' }, if: :specifies_delay? + validates :start_in, duration: { limit: '1 week' }, if: :specifies_delay? with_options allow_nil: true do validates :if, expression: true diff --git a/lib/gitlab/ci/config/entry/trigger.rb b/lib/gitlab/ci/config/entry/trigger.rb index c6ba53adfd7..0f94b3f94fe 100644 --- a/lib/gitlab/ci/config/entry/trigger.rb +++ b/lib/gitlab/ci/config/entry/trigger.rb @@ -5,12 +5,13 @@ module Gitlab class Config module Entry ## - # Entry that represents a cross-project downstream trigger. + # Entry that represents a parent-child or cross-project downstream trigger. # class Trigger < ::Gitlab::Config::Entry::Simplifiable strategy :SimpleTrigger, if: -> (config) { config.is_a?(String) } strategy :ComplexTrigger, if: -> (config) { config.is_a?(Hash) } + # cross-project class SimpleTrigger < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable @@ -28,11 +29,13 @@ module Gitlab config.key?(:include) end + # cross-project class CrossProjectTrigger < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Configurable - ALLOWED_KEYS = %i[project branch strategy].freeze + ALLOWED_KEYS = %i[project branch strategy forward].freeze attributes :project, :branch, :strategy validations do @@ -42,15 +45,26 @@ module Gitlab validates :branch, type: String, allow_nil: true validates :strategy, type: String, inclusion: { in: %w[depend], message: 'should be depend' }, allow_nil: true end + + entry :forward, ::Gitlab::Ci::Config::Entry::Trigger::Forward, + description: 'List what to forward to downstream pipelines' + + def value + { project: project, + branch: branch, + strategy: strategy, + forward: forward_value }.compact + end end + # parent-child class SameProjectTrigger < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable include ::Gitlab::Config::Entry::Configurable INCLUDE_MAX_SIZE = 3 - ALLOWED_KEYS = %i[strategy include].freeze + ALLOWED_KEYS = %i[strategy include forward].freeze attributes :strategy validations do @@ -64,8 +78,13 @@ module Gitlab reserved: true, metadata: { max_size: INCLUDE_MAX_SIZE } + entry :forward, ::Gitlab::Ci::Config::Entry::Trigger::Forward, + description: 'List what to forward to downstream pipelines' + def value - @config + { include: @config[:include], + strategy: strategy, + forward: forward_value }.compact end end diff --git a/lib/gitlab/ci/config/entry/trigger/forward.rb b/lib/gitlab/ci/config/entry/trigger/forward.rb new file mode 100644 index 00000000000..f80f018f149 --- /dev/null +++ b/lib/gitlab/ci/config/entry/trigger/forward.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents the configuration for passing attributes to the downstream pipeline + # + class Trigger + class Forward < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[yaml_variables pipeline_variables].freeze + + attributes ALLOWED_KEYS + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :yaml_variables, boolean: true + validates :pipeline_variables, boolean: true + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index fdb3e1b00f9..3839c43bd53 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -33,6 +33,10 @@ module Gitlab def fetch_local_content context.project.repository.blob_data_at(context.sha, location) + rescue GRPC::InvalidArgument + errors.push("Sha #{context.sha} is not valid!") + + nil end override :expand_context_attrs diff --git a/lib/gitlab/ci/config/yaml/tags/reference.rb b/lib/gitlab/ci/config/yaml/tags/reference.rb index 22822614b67..45787077c91 100644 --- a/lib/gitlab/ci/config/yaml/tags/reference.rb +++ b/lib/gitlab/ci/config/yaml/tags/reference.rb @@ -27,7 +27,7 @@ module Gitlab override :_resolve def _resolve(resolver) - object = resolver.config.dig(*location) + object = config_at_location(resolver) value = resolver.deep_resolve(object) raise MissingReferenceError, missing_ref_error_message unless value @@ -35,6 +35,12 @@ module Gitlab value end + def config_at_location(resolver) + resolver.config.dig(*location) + rescue TypeError + raise MissingReferenceError, missing_ref_error_message + end + def missing_ref_error_message "#{data[:tag]} #{data[:seq].inspect} could not be found" end diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb index d6b3af674a6..6041907ef78 100644 --- a/lib/gitlab/ci/parsers/coverage/cobertura.rb +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -8,140 +8,8 @@ module Gitlab InvalidXMLError = Class.new(Gitlab::Ci::Parsers::ParserError) InvalidLineInformationError = Class.new(Gitlab::Ci::Parsers::ParserError) - GO_SOURCE_PATTERN = '/usr/local/go/src' - MAX_SOURCES = 100 - def parse!(xml_data, coverage_report, project_path: nil, worktree_paths: nil) - root = Hash.from_xml(xml_data) - - context = { - project_path: project_path, - paths: worktree_paths&.to_set, - sources: [] - } - - parse_all(root, coverage_report, context) - rescue Nokogiri::XML::SyntaxError - raise InvalidXMLError, "XML parsing failed" - end - - private - - def parse_all(root, coverage_report, context) - return unless root.present? - - root.each do |key, value| - parse_node(key, value, coverage_report, context) - end - end - - def parse_node(key, value, coverage_report, context) - if key == 'sources' && value && value['source'].present? - parse_sources(value['source'], context) - elsif key == 'package' - Array.wrap(value).each do |item| - parse_package(item, coverage_report, context) - end - elsif key == 'class' - # This means the cobertura XML does not have classes within package nodes. - # This is possible in some cases like in simple JS project structures - # running Jest. - Array.wrap(value).each do |item| - parse_class(item, coverage_report, context) - end - elsif value.is_a?(Hash) - parse_all(value, coverage_report, context) - elsif value.is_a?(Array) - value.each do |item| - parse_all(item, coverage_report, context) - end - end - end - - def parse_sources(sources, context) - return unless context[:project_path] && context[:paths] - - sources = Array.wrap(sources) - - # TODO: Go cobertura has a different format with how their packages - # are included in the filename. So we can't rely on the sources. - # We'll deal with this later. - return if sources.include?(GO_SOURCE_PATTERN) - - sources.each do |source| - source = build_source_path(source, context) - context[:sources] << source if source.present? - end - end - - def build_source_path(source, context) - # | raw source | extracted | - # |-----------------------------|------------| - # | /builds/foo/test/SampleLib/ | SampleLib/ | - # | /builds/foo/test/something | something | - # | /builds/foo/test/ | nil | - # | /builds/foo/test | nil | - source.split("#{context[:project_path]}/", 2)[1] - end - - def parse_package(package, coverage_report, context) - classes = package.dig('classes', 'class') - return unless classes.present? - - matched_filenames = Array.wrap(classes).map do |item| - parse_class(item, coverage_report, context) - end - - # Remove these filenames from the paths to avoid conflict - # with other packages that may contain the same class filenames - remove_matched_filenames(matched_filenames, context) - end - - def remove_matched_filenames(filenames, context) - return unless context[:paths] - - filenames.each { |f| context[:paths].delete(f) } - end - - def parse_class(file, coverage_report, context) - return unless file["filename"].present? && file["lines"].present? - - parsed_lines = parse_lines(file["lines"]) - filename = determine_filename(file["filename"], context) - - coverage_report.add_file(filename, Hash[parsed_lines]) if filename - - filename - end - - def parse_lines(lines) - line_array = Array.wrap(lines["line"]) - - line_array.map do |line| - # Using `Integer()` here to raise exception on invalid values - [Integer(line["number"]), Integer(line["hits"])] - end - rescue StandardError - raise InvalidLineInformationError, "Line information had invalid values" - end - - def determine_filename(filename, context) - return filename unless context[:sources].any? - - full_filename = nil - - context[:sources].each_with_index do |source, index| - break if index >= MAX_SOURCES - break if full_filename = check_source(source, filename, context) - end - - full_filename - end - - def check_source(source, filename, context) - full_path = File.join(source, filename) - - return full_path if context[:paths].include?(full_path) + Nokogiri::XML::SAX::Parser.new(SaxDocument.new(coverage_report, project_path, worktree_paths)).parse(xml_data) end end end diff --git a/lib/gitlab/ci/parsers/coverage/sax_document.rb b/lib/gitlab/ci/parsers/coverage/sax_document.rb new file mode 100644 index 00000000000..27cce0e3a3b --- /dev/null +++ b/lib/gitlab/ci/parsers/coverage/sax_document.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Coverage + class SaxDocument < Nokogiri::XML::SAX::Document + GO_SOURCE_PATTERN = '/usr/local/go/src' + MAX_SOURCES = 100 + + def initialize(coverage_report, project_path, worktree_paths) + @coverage_report = coverage_report + @project_path = project_path + @paths = worktree_paths&.to_set + + @matched_filenames = [] + @parsed_lines = [] + @sources = [] + end + + def error(error) + raise Cobertura::InvalidXMLError, "XML parsing failed with error: #{error}" + end + + def start_element(node_name, attrs = []) + return unless node_name + + self.node_name = node_name + node_attrs = Hash[attrs] + + if node_name == 'class' && node_attrs["filename"].present? + self.filename = determine_filename(node_attrs["filename"]) + self.matched_filenames << filename if filename + elsif node_name == 'line' + self.parsed_lines << parse_line(node_attrs) + end + end + + def characters(node_content) + if node_name == 'source' + parse_source(node_content) + end + end + + def end_element(node_name) + if node_name == "package" + remove_matched_filenames + elsif node_name == "class" && filename && parsed_lines.present? + coverage_report.add_file(filename, Hash[parsed_lines]) + self.filename = nil + self.parsed_lines = [] + end + end + + private + + attr_accessor :coverage_report, :project_path, :paths, :sources, :node_name, :filename, :parsed_lines, :matched_filenames + + def parse_line(line) + [Integer(line["number"]), Integer(line["hits"])] + rescue StandardError + raise Cobertura::InvalidLineInformationError, "Line information had invalid values" + end + + def parse_source(node) + return unless project_path && paths && !node.include?(GO_SOURCE_PATTERN) + + source = build_source_path(node) + self.sources << source if source.present? + end + + def build_source_path(node) + # | raw source | extracted | + # |-----------------------------|------------| + # | /builds/foo/test/SampleLib/ | SampleLib/ | + # | /builds/foo/test/something | something | + # | /builds/foo/test/ | nil | + # | /builds/foo/test | nil | + node.split("#{project_path}/", 2)[1] + end + + def remove_matched_filenames + return unless paths + + matched_filenames.each { |f| paths.delete(f) } + end + + def determine_filename(filename) + return filename unless sources.any? + + full_filename = nil + + sources.each_with_index do |source, index| + break if index >= MAX_SOURCES + break if full_filename = check_source(source, filename) + end + + full_filename + end + + def check_source(source, filename) + full_path = File.join(source, filename) + + return full_path if paths.include?(full_path) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 9aec615d012..7baae2f53d7 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -19,6 +19,8 @@ module Gitlab end def parse! + set_report_version + return report_data unless valid? raise SecurityReportParserError, "Invalid report format" unless report_data.is_a?(Hash) @@ -26,7 +28,6 @@ module Gitlab create_scanner create_scan create_analyzer - set_report_version create_findings @@ -42,14 +43,19 @@ module Gitlab attr_reader :json_data, :report, :validate def valid? - if Feature.enabled?(:enforce_security_report_validation) - if !validate || schema_validator.valid? - report.schema_validation_status = :valid_schema - true + if Feature.enabled?(:show_report_validation_warnings, default_enabled: :yaml) + # We want validation to happen regardless of VALIDATE_SCHEMA CI variable + schema_validation_passed = schema_validator.valid? + + if validate + schema_validator.errors.each { |error| report.add_error('Schema', error) } unless schema_validation_passed + + schema_validation_passed else - report.schema_validation_status = :invalid_schema - schema_validator.errors.each { |error| report.add_error('Schema', error) } - false + # We treat all schema validation errors as warnings + schema_validator.errors.each { |error| report.add_warning('Schema', error) } + + true end else return true if !validate || schema_validator.valid? @@ -61,7 +67,7 @@ module Gitlab end def schema_validator - @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data) + @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data, report.version) end def report_data @@ -99,6 +105,7 @@ module Gitlab flags = create_flags(data['flags']) links = create_links(data['links']) location = create_location(data['location'] || {}) + evidence = create_evidence(data['evidence']) signatures = create_signatures(tracking_data(data)) if @vulnerability_finding_signatures_enabled && !signatures.empty? @@ -117,6 +124,7 @@ module Gitlab name: finding_name(data, identifiers, location), compare_key: data['cve'] || '', location: location, + evidence: evidence, severity: parse_severity_level(data['severity']), confidence: parse_confidence_level(data['confidence']), scanner: create_scanner(data['scanner']), @@ -253,6 +261,12 @@ module Gitlab raise NotImplementedError end + def create_evidence(evidence_data) + return unless evidence_data.is_a?(Hash) + + ::Gitlab::Ci::Reports::Security::Evidence.new(data: evidence_data) + end + def finding_name(data, identifiers, location) return data['message'] if data['message'].present? return data['name'] if data['name'].present? diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 651ed23eb25..0ab1a128052 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -6,20 +6,56 @@ module Gitlab module Security module Validators class SchemaValidator + # https://docs.gitlab.com/ee/update/deprecations.html#147 + SUPPORTED_VERSIONS = { + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0] + }.freeze + + # https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/tags + PREVIOUS_RELEASES = %w[10.0.0 12.0.0 12.1.0 13.0.0 + 13.1.0 2.3.0-rc1 2.3.0-rc1 2.3.1-rc1 2.3.2-rc1 2.3.3-rc1 + 2.4.0-rc1 3.0.0 3.0.0-rc1 3.1.0-rc1 4.0.0-rc1 5.0.0-rc1 + 5.0.1-rc1 6.0.0-rc1 6.0.1-rc1 6.1.0-rc1 7.0.0-rc1 7.0.1-rc1 + 8.0.0-rc1 8.0.1-rc1 8.1.0-rc1 9.0.0-rc1].freeze + + # These come from https://app.periscopedata.com/app/gitlab/895813/Secure-Scan-metrics?widget=12248944&udv=1385516 + KNOWN_VERSIONS_TO_DEPRECATE = %w[0.1 1.0 1.0.0 1.2 1.3 10.0.0 12.1.0 13.1.0 2.0 2.1 2.1.0 2.3 2.3.0 2.4 3.0 3.0.0 3.0.6 3.13.2 V2.7.0].freeze + + VERSIONS_TO_DEPRECATE_IN_15_0 = (PREVIOUS_RELEASES + KNOWN_VERSIONS_TO_DEPRECATE).freeze + + DEPRECATED_VERSIONS = { + cluster_image_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, + container_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, + coverage_fuzzing: VERSIONS_TO_DEPRECATE_IN_15_0, + dast: VERSIONS_TO_DEPRECATE_IN_15_0, + api_fuzzing: VERSIONS_TO_DEPRECATE_IN_15_0, + dependency_scanning: VERSIONS_TO_DEPRECATE_IN_15_0, + sast: VERSIONS_TO_DEPRECATE_IN_15_0, + secret_detection: VERSIONS_TO_DEPRECATE_IN_15_0 + }.freeze + class Schema def root_path File.join(__dir__, 'schemas') end - def initialize(report_type) + def initialize(report_type, report_version) @report_type = report_type.to_sym + @report_version = report_version.to_s end delegate :validate, to: :schemer private - attr_reader :report_type + attr_reader :report_type, :report_version def schemer JSONSchemer.schema(pathname) @@ -30,7 +66,19 @@ module Gitlab end def schema_path - File.join(root_path, file_name) + # We can't exactly error out here pre-15.0. + # If the report itself doesn't specify the schema version, + # it will be considered invalid post-15.0 but for now we will + # validate against earliest supported version. + # https://gitlab.com/gitlab-org/gitlab/-/issues/335789#note_801479803 + # describes the indended behavior in detail + # TODO: After 15.0 - pass report_type and report_data here and + # error out if no version. + report_declared_version = File.join(root_path, report_version, file_name) + return report_declared_version if File.file?(report_declared_version) + + earliest_supported_version = SUPPORTED_VERSIONS[report_type].min + File.join(root_path, earliest_supported_version, file_name) end def file_name @@ -38,9 +86,10 @@ module Gitlab end end - def initialize(report_type, report_data) + def initialize(report_type, report_data, report_version = nil) @report_type = report_type @report_data = report_data + @report_version = report_version end def valid? @@ -53,10 +102,10 @@ module Gitlab private - attr_reader :report_type, :report_data + attr_reader :report_type, :report_data, :report_version def schema - Schema.new(report_type) + Schema.new(report_type, report_version) end end end diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb index 54b54bd0514..71dfc1a676c 100644 --- a/lib/gitlab/ci/pipeline/chain/create.rb +++ b/lib/gitlab/ci/pipeline/chain/create.rb @@ -14,7 +14,7 @@ module Gitlab with_bulk_insert_tags do pipeline.transaction do pipeline.save! - CommitStatus.bulk_insert_tags!(statuses) if bulk_insert_tags? + CommitStatus.bulk_insert_tags!(statuses) end end end @@ -29,15 +29,9 @@ module Gitlab private - def bulk_insert_tags? - strong_memoize(:bulk_insert_tags) do - ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml) - end - end - def with_bulk_insert_tags previous = Thread.current['ci_bulk_insert_tags'] - Thread.current['ci_bulk_insert_tags'] = bulk_insert_tags? + Thread.current['ci_bulk_insert_tags'] = true yield ensure Thread.current['ci_bulk_insert_tags'] = previous diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb index 10c0fe295f8..ee6c3898592 100644 --- a/lib/gitlab/ci/pipeline/logger.rb +++ b/lib/gitlab/ci/pipeline/logger.rb @@ -94,6 +94,7 @@ module Gitlab private attr_reader :project, :destination, :started_at, :log_conditions + delegate :current_monotonic_time, to: :class def age diff --git a/lib/gitlab/ci/reports/security/evidence.rb b/lib/gitlab/ci/reports/security/evidence.rb new file mode 100644 index 00000000000..a19f52f7195 --- /dev/null +++ b/lib/gitlab/ci/reports/security/evidence.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Evidence + attr_reader :data + + def initialize(data:) + @data = data + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index 69fb8474cde..911a7f5d358 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -13,6 +13,7 @@ module Gitlab attr_reader :flags attr_reader :links attr_reader :location + attr_reader :evidence attr_reader :metadata_version attr_reader :name attr_reader :old_location @@ -33,13 +34,14 @@ module Gitlab alias_method :cve, :compare_key - def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists + def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists @compare_key = compare_key @confidence = confidence @identifiers = identifiers @flags = flags @links = links @location = location + @evidence = evidence @metadata_version = metadata_version @name = name @original_data = original_data @@ -65,6 +67,7 @@ module Gitlab flags links location + evidence metadata_version name project_fingerprint diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index fbf8c81ac36..8c528056d0c 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -6,7 +6,7 @@ module Gitlab module Security class Report attr_reader :created_at, :type, :pipeline, :findings, :scanners, :identifiers - attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status + attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status, :warnings delegate :project_id, to: :pipeline @@ -19,6 +19,7 @@ module Gitlab @identifiers = {} @scanned_resources = [] @errors = [] + @warnings = [] end def commit_sha @@ -29,6 +30,10 @@ module Gitlab errors << { type: type, message: message } end + def add_warning(type, message) + warnings << { type: type, message: message } + end + def errored? errors.present? end diff --git a/lib/gitlab/ci/reports/test_suite_comparer.rb b/lib/gitlab/ci/reports/test_suite_comparer.rb index 287a03cefe2..7fa744d047c 100644 --- a/lib/gitlab/ci/reports/test_suite_comparer.rb +++ b/lib/gitlab/ci/reports/test_suite_comparer.rb @@ -106,7 +106,7 @@ module Gitlab private def max_tests(*used) - [DEFAULT_MAX_TESTS - used.map(&:count).sum, DEFAULT_MIN_TESTS].max + [DEFAULT_MAX_TESTS - used.sum(&:count), DEFAULT_MIN_TESTS].max end end end diff --git a/lib/gitlab/ci/status/build/waiting_for_approval.rb b/lib/gitlab/ci/status/build/waiting_for_approval.rb index 59869a947a9..ac3f5838d26 100644 --- a/lib/gitlab/ci/status/build/waiting_for_approval.rb +++ b/lib/gitlab/ci/status/build/waiting_for_approval.rb @@ -9,11 +9,35 @@ module Gitlab { image: 'illustrations/manual_action.svg', size: 'svg-394', - title: 'Waiting for approval', - content: "This job deploys to the protected environment \"#{subject.deployment&.environment&.name}\" which requires approvals. Use the Deployments API to approve or reject the deployment." + title: _('Waiting for approval'), + content: _("This job deploys to the protected environment \"%{environment}\" which requires approvals.") % { environment: subject.deployment&.environment&.name } } end + def has_action? + true + end + + def action_icon + nil + end + + def action_title + nil + end + + def action_button_title + _('Go to environments page to approve or reject') + end + + def action_path + project_environments_path(subject.project) + end + + def action_method + :get + end + def self.matches?(build, user) build.waiting_for_deployment_approval? end diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 64e3b695e27..bbe1b0a4b82 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -4,8 +4,14 @@ # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml # Read more about how to use this script on this blog post https://about.gitlab.com/2019/01/28/android-publishing-with-gitlab-and-fastlane/ -# You will also need to configure your build.gradle, Dockerfile, and fastlane configuration to make this work. # If you are looking for a simpler template that does not publish, see the Android template. +# You will also need to configure your build.gradle, Dockerfile, and fastlane configuration to make this work. + +# The following environment variables also need to be defined via the CI/CD settings: +# +# - $signing_jks_file_hex: A hex-encoded Java keystore file containing your signing keys. +# To encode this file, use `xxd -p <your-keystore-file>.jks` and save the output as `$signing_jks_file_hex` +# - $google_play_service_account_api_key_json: Your Google Play service account credentials - https://docs.fastlane.tools/getting-started/android/setup/#collect-your-google-credentials stages: - environment @@ -41,20 +47,21 @@ ensureContainer: before_script: - "mkdir -p ~/.docker && echo '{\"experimental\": \"enabled\"}' > ~/.docker/config.json" - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - # Skip update container `script` if the container already exists - # via https://gitlab.com/gitlab-org/gitlab-foss/issues/26866#note_97609397 -> https://stackoverflow.com/a/52077071/796832 - - docker manifest inspect $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG > /dev/null && exit || true - + - | + if docker manifest inspect $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG > /dev/null; then + echo 'Skipping job since there is already an image with this tag' + exit 0 + fi .build_job: image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG stage: build before_script: - # We store this binary file in a variable as hex with this command: `xxd -p android-app.jks` + # We store this binary file in a project variable as hex with this command: `xxd -p android-app.jks` # Then we convert the hex back to a binary file - echo "$signing_jks_file_hex" | xxd -r -p - > android-signing-keystore.jks - - "export VERSION_CODE=$CI_PIPELINE_IID && echo $VERSION_CODE" - - "export VERSION_SHA=`echo ${CI_COMMIT_SHA:0:8}` && echo $VERSION_SHA" + - export VERSION_CODE="$CI_PIPELINE_IID" && echo "$VERSION_CODE" + - export VERSION_SHA="${CI_COMMIT_SHA:0:8}" && echo "$VERSION_SHA" after_script: - rm -f android-signing-keystore.jks || true artifacts: diff --git a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml index a50e722f18a..6354db38f58 100644 --- a/lib/gitlab/ci/templates/Dart.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Dart.gitlab-ci.yml @@ -18,7 +18,7 @@ cache: - .pub-cache/global_packages before_script: - - export PATH="$PATH":"~/.pub-cache/bin" + - export PATH="$PATH:$HOME/.pub-cache/bin" - pub get --no-precompile test: diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml index d176ce19299..a5c261e367a 100644 --- a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: image: "cirrusci/flutter:1.22.5" before_script: - pub global activate dart_code_metrics - - export PATH="$PATH":"$HOME/.pub-cache/bin" + - export PATH="$PATH:$HOME/.pub-cache/bin" script: - metrics lib -r codeclimate > gl-code-quality-report.json artifacts: @@ -20,7 +20,7 @@ test: image: "cirrusci/flutter:1.22.5" before_script: - pub global activate junitreport - - export PATH="$PATH":"$HOME/.pub-cache/bin" + - export PATH="$PATH:$HOME/.pub-cache/bin" script: - flutter test --machine --coverage | tojunit -o report.xml - lcov --summary coverage/lcov.info diff --git a/lib/gitlab/ci/templates/Go.gitlab-ci.yml b/lib/gitlab/ci/templates/Go.gitlab-ci.yml index b5dd0005013..19e4ffdbe1e 100644 --- a/lib/gitlab/ci/templates/Go.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Go.gitlab-ci.yml @@ -16,9 +16,9 @@ variables: # repository in /go/src/gitlab.com/namespace/project # Thus, making a symbolic link corrects this. before_script: - - mkdir -p $GOPATH/src/$(dirname $REPO_NAME) - - ln -svf $CI_PROJECT_DIR $GOPATH/src/$REPO_NAME - - cd $GOPATH/src/$REPO_NAME + - mkdir -p "$GOPATH/src/$(dirname $REPO_NAME)" + - ln -svf "$CI_PROJECT_DIR" "$GOPATH/src/$REPO_NAME" + - cd "$GOPATH/src/$REPO_NAME" stages: - test diff --git a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml index 76f0c9f8427..08dc10d34b7 100644 --- a/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Gradle.gitlab-ci.yml @@ -17,7 +17,8 @@ variables: GRADLE_OPTS: "-Dorg.gradle.daemon=false" before_script: - - export GRADLE_USER_HOME=`pwd`/.gradle + - GRADLE_USER_HOME="$(pwd)/.gradle" + - export GRADLE_USER_HOME build: stage: build diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index 3c514d7b0c6..7e59354c4a1 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -23,8 +23,8 @@ variables: before_script: - apt-get update -qq && apt-get install -y -qq unzip - curl -sSL https://get.sdkman.io | bash - - echo sdkman_auto_answer=true > /root/.sdkman/etc/config - - source /root/.sdkman/bin/sdkman-init.sh + - echo sdkman_auto_answer=true > ~/.sdkman/etc/config + - source ~/.sdkman/bin/sdkman-init.sh - sdk install gradle $GRADLE_VERSION < /dev/null - sdk use gradle $GRADLE_VERSION # As it's not a good idea to version gradle.properties feel free to add your @@ -36,7 +36,7 @@ before_script: # Be aware that if you are using Angular profile, # Bower cannot be run as root if you don't allow it before. # Feel free to remove next line if you are not using Bower - - echo {\"allow_root\":true} > /root/.bowerrc + - echo '{"allow_root":true}' > ~/.bowerrc # This build job does the full grails pipeline # (compile, test, integrationTest, war, assemble). diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 99fd9870b1d..d1018f1e769 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -2,7 +2,7 @@ browser_performance: stage: performance - image: docker:19.03.12 + image: docker:20.10.12 allow_failure: true variables: DOCKER_TLS_CERTDIR: "" @@ -10,19 +10,21 @@ browser_performance: SITESPEED_VERSION: 14.1.0 SITESPEED_OPTIONS: '' services: - - docker:19.03.12-dind + - name: 'docker:20.10.12-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + if [ -z "$DOCKER_HOST" ] && [ -n "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi - - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) + - CI_ENVIRONMENT_URL="$(cat environment_url.txt)" + - export CI_ENVIRONMENT_URL - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. - - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - (env | grep -i _proxy= >/dev/null 2>&1) && apk --no-cache add wget - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml index 99fd9870b1d..bb7e020b159 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -2,7 +2,7 @@ browser_performance: stage: performance - image: docker:19.03.12 + image: docker:20.10.12 allow_failure: true variables: DOCKER_TLS_CERTDIR: "" @@ -10,11 +10,12 @@ browser_performance: SITESPEED_VERSION: 14.1.0 SITESPEED_OPTIONS: '' services: - - docker:19.03.12-dind + - name: 'docker:20.10.12-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + if [ -z "$DOCKER_HOST" ] && [ -n "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi @@ -22,7 +23,7 @@ browser_performance: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. - - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - (env | grep -i _proxy= >/dev/null 2>&1) && apk --no-cache add wget - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index d5ca93a0a3b..f3d2e293c86 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.5.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.9.1' build: stage: build @@ -7,7 +7,7 @@ build: variables: DOCKER_TLS_CERTDIR: '' services: - - name: 'docker:20.10.6-dind' + - name: 'docker:20.10.12-dind' command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index d5ca93a0a3b..f3d2e293c86 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.5.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.9.1' build: stage: build @@ -7,7 +7,7 @@ build: variables: DOCKER_TLS_CERTDIR: '' services: - - name: 'docker:20.10.6-dind' + - name: 'docker:20.10.12-dind' command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 6942631a97f..6a95d042842 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -1,9 +1,10 @@ code_quality: stage: test - image: docker:19.03.12 + image: docker:20.10.12 allow_failure: true services: - - docker:19.03.12-dind + - name: 'docker:20.10.12-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" @@ -13,7 +14,7 @@ code_quality: - export SOURCE_CODE=$PWD - | if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + if [ -z "$DOCKER_HOST" ] && [ -n "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 28ac627f103..cc204207f84 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.17.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index 65c9232f3b9..1a99db67441 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -11,7 +11,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 075e13e87f0..bc4f2099d94 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.18.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index e9c5d970c21..ce584091eab 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.18.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.22.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml index fc51f5adb3c..89a44eddefd 100644 --- a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml @@ -11,7 +11,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. LICENSE_MANAGEMENT_VERSION: 3 diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index 8e34388893a..eea1c397108 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -1,6 +1,6 @@ load_performance: stage: performance - image: docker:19.03.11 + image: docker:20.10.12 allow_failure: true variables: DOCKER_TLS_CERTDIR: "" @@ -10,11 +10,12 @@ load_performance: K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' services: - - docker:19.03.11-dind + - name: 'docker:20.10.12-dind' + command: ['--tls=false', '--host=tcp://0.0.0.0:2375'] script: - | if ! docker info &>/dev/null; then - if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then + if [ -z "$DOCKER_HOST" ] && [ -n "$KUBERNETES_PORT" ]; then export DOCKER_HOST='tcp://localhost:2375' fi fi diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index fa7f6ffa2b7..5ddfb2a54be 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -1,7 +1,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" iac-sast: diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 25d20563010..8cc9ea0200c 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -6,7 +6,7 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SAST_EXCLUDED_ANALYZERS: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index 4e4f96bc7c7..0ef6f63bb94 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -5,7 +5,7 @@ # How to set: https://docs.gitlab.com/ee/ci/yaml/#variables variables: - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SECRETS_ANALYZER_VERSION: "3" SECRET_DETECTION_EXCLUDED_PATHS: "" @@ -30,24 +30,43 @@ secret_detection: - if: $CI_COMMIT_BRANCH script: - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - - if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi + # Historic scan - | - # we don't need the whole history when excluding in the next `git fetch` line, - # so git depth=1 - git fetch origin --depth=1 $CI_DEFAULT_BRANCH - # shallow clone $CI_COMMIT_REF_NAME to get commits associated with MR or push - git fetch --shallow-exclude=${CI_DEFAULT_BRANCH} origin $CI_COMMIT_REF_NAME - # determine what commits we need to scan using "git log A..B" - git log --no-merges --pretty=format:"%H" refs/remotes/origin/${CI_DEFAULT_BRANCH}..refs/remotes/origin/${CI_COMMIT_REF_NAME} >${CI_COMMIT_SHA}_commit_list.txt - - # we need to extend the git fetch depth to the number of commits + 2 for the following reasons: - # because busybox wc only counts \n and there is no trailing \n (+1) - # include the parent commit of the base commit in this MR/Push event. This is needed because - # `git diff -p` needs something to compare changes in that commit against (+1) - git fetch --depth=$(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 2)) origin $CI_COMMIT_REF_NAME + if [ "$SECRET_DETECTION_HISTORIC_SCAN" == "true" ] + then + echo "historic scan" + git fetch --unshallow origin $CI_COMMIT_REF_NAME + /analyzer run + exit + fi + # Default branch scan + - if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit; fi + # Push event + - | + if [ "$CI_COMMIT_BEFORE_SHA" == "0000000000000000000000000000000000000000" ]; + then + # first commit on a new branch + echo ${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt + git fetch --depth=2 origin $CI_COMMIT_REF_NAME + else + # determine commit range so that we can fetch the appropriate depth + # check the exit code to determine if we need to limit the commit_list.txt to CI_COMMIT_SHA. + if ! git log --pretty=format:"%H" ${CI_COMMIT_BEFORE_SHA}..${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt; + then + echo "unable to determine commit range, limiting to ${CI_COMMIT_SHA}" + echo ${CI_COMMIT_SHA} >${CI_COMMIT_SHA}_commit_list.txt + else + # append newline to to list since `git log` does not end with a + # newline, this is to keep the log messages consistent + echo >> ${CI_COMMIT_SHA}_commit_list.txt + fi - # +1 because busybox wc only counts \n and there is no trailing \n - echo "scanning $(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 1)) commits" + # we need to extend the git fetch depth to the number of commits + 1 for the following reasons: + # to include the parent commit of the base commit in this MR/Push event. This is needed because + # `git diff -p` needs something to compare changes in that commit against + git fetch --depth=$(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 1)) origin $CI_COMMIT_REF_NAME + fi + echo "scanning $(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt))) commits for a push event" export SECRET_DETECTION_COMMITS_FILE=${CI_COMMIT_SHA}_commit_list.txt - /analyzer run - rm "$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml index d32444833fb..85f90984045 100644 --- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml @@ -8,7 +8,7 @@ pages: stage: deploy script: - mkdir .public - - cp -r * .public + - cp -r ./* .public - rm -rf public - mv .public public artifacts: diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index 4917abf6ae9..6ed5e05ed4c 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -47,7 +47,8 @@ run: pages: script: - pip install sphinx sphinx-rtd-theme - - cd doc ; make html + - cd doc + - make html - mv build/html/ ../public/ artifacts: paths: diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 1660a9250e3..33c0928db6f 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -52,7 +52,7 @@ rails: # This deploy job uses a simple deploy flow to Heroku, other providers, e.g. AWS Elastic Beanstalk # are supported too: https://github.com/travis-ci/dpl deploy: - type: deploy + stage: deploy environment: production script: - gem install dpl diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 009061ce844..aff8b6cb7fa 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -10,7 +10,7 @@ variables: FUZZAPI_VERSION: "1" - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION} apifuzzer_fuzz: diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index 01041f4f056..bd8ba71effe 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -10,7 +10,7 @@ variables: FUZZAPI_VERSION: "1" - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" FUZZAPI_IMAGE: api-fuzzing apifuzzer_fuzz: diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index a2933085d4e..d82f9f06f8d 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -24,7 +24,7 @@ variables: # Setting this variable affects all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # DAST_API_VERSION: "1" DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml index 57f1993921d..0e0afa489a3 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -24,7 +24,7 @@ variables: # Setting this variable affects all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" # DAST_API_VERSION: "1" DAST_API_IMAGE: api-fuzzing diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml index 7ffec7d2e6b..3f9c87b7abf 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml @@ -5,7 +5,7 @@ stages: - dast variables: - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" DAST_API_VERSION: "1" DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml index 3e7ab9b5c3b..998425aa141 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -13,7 +13,7 @@ variables: DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" dast: stage: dast diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index 0ecbe5e14b8..e8e7fe62e70 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -25,7 +25,7 @@ variables: DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" dast: stage: dast diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 3d07674c377..c755211ec11 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -25,7 +25,7 @@ variables: DAST_VERSION: 2 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) - SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" dast: stage: dast diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index 82c7bfd0620..a6fd070ec34 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -14,8 +14,11 @@ # Docs: https://docs.gitlab.com/ee/topics/airgap/ variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/security-products" SECURE_BINARIES_ANALYZERS: >- - bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, + bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kics, kubesec, semgrep, bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, license-finder, dast, dast-runner-validation, api-fuzzing @@ -40,7 +43,7 @@ variables: script: - docker info - env - - if [ -z "$SECURE_BINARIES_IMAGE" ]; then export SECURE_BINARIES_IMAGE=${SECURE_BINARIES_IMAGE:-"registry.gitlab.com/gitlab-org/security-products/analyzers/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"}; fi + - if [ -z "$SECURE_BINARIES_IMAGE" ]; then export SECURE_BINARIES_IMAGE=${SECURE_BINARIES_IMAGE:-"${SECURE_ANALYZERS_PREFIX}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"}; fi - docker pull --quiet ${SECURE_BINARIES_IMAGE} - mkdir -p output/$(dirname ${CI_JOB_NAME}) - | diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index e696c75253e..84a962e1541 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -24,19 +24,19 @@ cache: .init: &init stage: init script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform init .validate: &validate stage: validate script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform validate .build: &build stage: build script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform plan - gitlab-terraform plan-json artifacts: @@ -48,7 +48,7 @@ cache: .deploy: &deploy stage: deploy script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform apply when: manual only: @@ -58,6 +58,6 @@ cache: .destroy: &destroy stage: cleanup script: - - cd ${TF_ROOT} + - cd "${TF_ROOT}" - gitlab-terraform destroy when: manual diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml index 8f4a836441d..5ea2bc07ffa 100644 --- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml @@ -14,7 +14,8 @@ stages: a11y: stage: accessibility image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.1.1 - script: /gitlab-accessibility.sh $a11y_urls + script: + - /gitlab-accessibility.sh "$a11y_urls" allow_failure: true artifacts: when: always diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index e0df9799917..2349c37c130 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -25,7 +25,7 @@ browser_performance: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. - - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - (env | grep -i _proxy= >/dev/null 2>&1) && apk --no-cache add wget - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml index ad24ebae8d4..73ab5fcbe44 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -25,7 +25,7 @@ browser_performance: - mkdir gitlab-exporter # Busybox wget does not support proxied HTTPS, get the real thing. # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. - - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget + - (env | grep -i _proxy= >/dev/null 2>&1) && apk --no-cache add wget - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | diff --git a/lib/gitlab/ci/trace/remote_checksum.rb b/lib/gitlab/ci/trace/remote_checksum.rb index 7f43d91e6d7..eaa9be9dd15 100644 --- a/lib/gitlab/ci/trace/remote_checksum.rb +++ b/lib/gitlab/ci/trace/remote_checksum.rb @@ -23,6 +23,7 @@ module Gitlab private attr_reader :trace_artifact + delegate :aws?, :google?, to: :object_store_config, prefix: :provider def fetch_md5_checksum diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 9ef6e7f5fa9..bfcf67693e7 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -10,6 +10,7 @@ module Gitlab @pipeline = pipeline @instance_variables_builder = Builder::Instance.new @project_variables_builder = Builder::Project.new(project) + @group_variables_builder = Builder::Group.new(project.group) end def scoped_variables(job, environment:, dependencies:) @@ -18,8 +19,7 @@ module Gitlab variables.concat(project.predefined_variables) variables.concat(pipeline.predefined_variables) variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner - variables.concat(kubernetes_variables(job)) - variables.concat(deployment_variables(environment: environment, job: job)) + variables.concat(kubernetes_variables(environment: environment, job: job)) variables.concat(job.yaml_variables) variables.concat(user_variables(job.user)) variables.concat(job.dependency_variables) if dependencies @@ -32,11 +32,15 @@ module Gitlab end end - def kubernetes_variables(job) + def kubernetes_variables(environment:, job:) ::Gitlab::Ci::Variables::Collection.new.tap do |collection| - # Should get merged with the cluster kubeconfig in deployment_variables, see - # https://gitlab.com/gitlab-org/gitlab/-/issues/335089 + # NOTE: deployment_variables will be removed as part of cleanup for + # https://gitlab.com/groups/gitlab-org/configure/-/epics/8 + # Until then, we need to make both the old and the new KUBECONFIG contexts available + collection.concat(deployment_variables(environment: environment, job: job)) template = ::Ci::GenerateKubeconfigService.new(job).execute + kubeconfig_yaml = collection['KUBECONFIG']&.value + template.merge_yaml(kubeconfig_yaml) if kubeconfig_yaml.present? if template.valid? collection.append(key: 'KUBECONFIG', value: template.to_yaml, public: false, file: true) @@ -72,9 +76,13 @@ module Gitlab end def secret_group_variables(environment:, ref:) - return [] unless project.group + if memoize_secret_variables? + memoized_secret_group_variables(environment: environment) + else + return [] unless project.group - project.group.ci_variables_for(ref, project, environment: environment) + project.group.ci_variables_for(ref, project, environment: environment) + end end def secret_project_variables(environment:, ref:) @@ -90,6 +98,8 @@ module Gitlab attr_reader :pipeline attr_reader :instance_variables_builder attr_reader :project_variables_builder + attr_reader :group_variables_builder + delegate :project, to: :pipeline def predefined_variables(job) @@ -119,6 +129,15 @@ module Gitlab end end + def memoized_secret_group_variables(environment:) + strong_memoize_with(:secret_group_variables, environment) do + group_variables_builder + .secret_variables( + environment: environment, + protected_ref: protected_ref?) + end + end + def ci_node_total_value(job) parallel = job.options&.dig(:parallel) parallel = parallel.dig(:total) if parallel.is_a?(Hash) diff --git a/lib/gitlab/ci/variables/builder/group.rb b/lib/gitlab/ci/variables/builder/group.rb new file mode 100644 index 00000000000..3f3e04038df --- /dev/null +++ b/lib/gitlab/ci/variables/builder/group.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Builder + class Group + include Gitlab::Utils::StrongMemoize + + def initialize(group) + @group = group + end + + def secret_variables(environment:, protected_ref: false) + return [] unless group + + variables = base_scope + variables = variables.unprotected unless protected_ref + variables = variables.for_environment(environment) + variables = variables.group_by(&:group_id) + variables = list_of_ids.reverse.flat_map { |group| variables[group.id] }.compact + Gitlab::Ci::Variables::Collection.new(variables) + end + + private + + attr_reader :group + + def base_scope + strong_memoize(:base_scope) do + ::Ci::GroupVariable.for_groups(list_of_ids) + end + end + + def list_of_ids + strong_memoize(:list_of_ids) do + if group.root_ancestor.use_traversal_ids? + [group] + group.ancestors(hierarchy_order: :asc) + else + [group] + group.ancestors + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 553508c8638..15ebd506055 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -45,7 +45,7 @@ module Gitlab validate_job!(name, job) end - YamlProcessor::Dag.check_circular_dependencies!(@jobs) + check_circular_dependencies end def validate_job!(name, job) @@ -146,6 +146,17 @@ module Gitlab end end + def check_circular_dependencies + jobs = @jobs.values.to_h do |job| + name = job[:name].to_s + needs = job.dig(:needs, :job).to_a + + [name, needs.map { |need| need[:name].to_s }] + end + + Dag.check_circular_dependencies!(jobs) + end + def error!(message) raise ValidationError, message end diff --git a/lib/gitlab/ci/yaml_processor/dag.rb b/lib/gitlab/ci/yaml_processor/dag.rb index 8ab9573dd20..4a122c73e80 100644 --- a/lib/gitlab/ci/yaml_processor/dag.rb +++ b/lib/gitlab/ci/yaml_processor/dag.rb @@ -7,28 +7,22 @@ module Gitlab class Dag include TSort - MissingNodeError = Class.new(StandardError) - def initialize(nodes) @nodes = nodes end - def self.check_circular_dependencies!(jobs) - nodes = jobs.values.to_h do |job| - name = job[:name].to_s - needs = job.dig(:needs, :job).to_a - - [name, needs.map { |need| need[:name].to_s }] - end + def self.order(jobs) + new(jobs).tsort + end - new(nodes).tsort + def self.check_circular_dependencies!(jobs) + new(jobs).tsort rescue TSort::Cyclic raise ValidationError, 'The pipeline has circular dependencies' - rescue MissingNodeError end def tsort_each_child(node, &block) - raise MissingNodeError, "node #{node} is missing" unless @nodes[node] + return unless @nodes[node] @nodes[node].each(&block) end diff --git a/lib/gitlab/color.rb b/lib/gitlab/color.rb new file mode 100644 index 00000000000..e0caabb0ec6 --- /dev/null +++ b/lib/gitlab/color.rb @@ -0,0 +1,222 @@ +# frozen_string_literal: true + +module Gitlab + class Color + PATTERN = /\A\#(?:[0-9A-Fa-f]{3}){1,2}\Z/.freeze + + def initialize(value) + @value = value&.strip&.freeze + end + + module Constants + DARK = Color.new('#333333') + LIGHT = Color.new('#FFFFFF') + + COLOR_NAME_TO_HEX = { + black: '#000000', + silver: '#C0C0C0', + gray: '#808080', + white: '#FFFFFF', + maroon: '#800000', + red: '#FF0000', + purple: '#800080', + fuchsia: '#FF00FF', + green: '#008000', + lime: '#00FF00', + olive: '#808000', + yellow: '#FFFF00', + navy: '#000080', + blue: '#0000FF', + teal: '#008080', + aqua: '#00FFFF', + orange: '#FFA500', + aliceblue: '#F0F8FF', + antiquewhite: '#FAEBD7', + aquamarine: '#7FFFD4', + azure: '#F0FFFF', + beige: '#F5F5DC', + bisque: '#FFE4C4', + blanchedalmond: '#FFEBCD', + blueviolet: '#8A2BE2', + brown: '#A52A2A', + burlywood: '#DEB887', + cadetblue: '#5F9EA0', + chartreuse: '#7FFF00', + chocolate: '#D2691E', + coral: '#FF7F50', + cornflowerblue: '#6495ED', + cornsilk: '#FFF8DC', + crimson: '#DC143C', + darkblue: '#00008B', + darkcyan: '#008B8B', + darkgoldenrod: '#B8860B', + darkgray: '#A9A9A9', + darkgreen: '#006400', + darkgrey: '#A9A9A9', + darkkhaki: '#BDB76B', + darkmagenta: '#8B008B', + darkolivegreen: '#556B2F', + darkorange: '#FF8C00', + darkorchid: '#9932CC', + darkred: '#8B0000', + darksalmon: '#E9967A', + darkseagreen: '#8FBC8F', + darkslateblue: '#483D8B', + darkslategray: '#2F4F4F', + darkslategrey: '#2F4F4F', + darkturquoise: '#00CED1', + darkviolet: '#9400D3', + deeppink: '#FF1493', + deepskyblue: '#00BFFF', + dimgray: '#696969', + dimgrey: '#696969', + dodgerblue: '#1E90FF', + firebrick: '#B22222', + floralwhite: '#FFFAF0', + forestgreen: '#228B22', + gainsboro: '#DCDCDC', + ghostwhite: '#F8F8FF', + gold: '#FFD700', + goldenrod: '#DAA520', + greenyellow: '#ADFF2F', + grey: '#808080', + honeydew: '#F0FFF0', + hotpink: '#FF69B4', + indianred: '#CD5C5C', + indigo: '#4B0082', + ivory: '#FFFFF0', + khaki: '#F0E68C', + lavender: '#E6E6FA', + lavenderblush: '#FFF0F5', + lawngreen: '#7CFC00', + lemonchiffon: '#FFFACD', + lightblue: '#ADD8E6', + lightcoral: '#F08080', + lightcyan: '#E0FFFF', + lightgoldenrodyellow: '#FAFAD2', + lightgray: '#D3D3D3', + lightgreen: '#90EE90', + lightgrey: '#D3D3D3', + lightpink: '#FFB6C1', + lightsalmon: '#FFA07A', + lightseagreen: '#20B2AA', + lightskyblue: '#87CEFA', + lightslategray: '#778899', + lightslategrey: '#778899', + lightsteelblue: '#B0C4DE', + lightyellow: '#FFFFE0', + limegreen: '#32CD32', + linen: '#FAF0E6', + mediumaquamarine: '#66CDAA', + mediumblue: '#0000CD', + mediumorchid: '#BA55D3', + mediumpurple: '#9370DB', + mediumseagreen: '#3CB371', + mediumslateblue: '#7B68EE', + mediumspringgreen: '#00FA9A', + mediumturquoise: '#48D1CC', + mediumvioletred: '#C71585', + midnightblue: '#191970', + mintcream: '#F5FFFA', + mistyrose: '#FFE4E1', + moccasin: '#FFE4B5', + navajowhite: '#FFDEAD', + oldlace: '#FDF5E6', + olivedrab: '#6B8E23', + orangered: '#FF4500', + orchid: '#DA70D6', + palegoldenrod: '#EEE8AA', + palegreen: '#98FB98', + paleturquoise: '#AFEEEE', + palevioletred: '#DB7093', + papayawhip: '#FFEFD5', + peachpuff: '#FFDAB9', + peru: '#CD853F', + pink: '#FFC0CB', + plum: '#DDA0DD', + powderblue: '#B0E0E6', + rosybrown: '#BC8F8F', + royalblue: '#4169E1', + saddlebrown: '#8B4513', + salmon: '#FA8072', + sandybrown: '#F4A460', + seagreen: '#2E8B57', + seashell: '#FFF5EE', + sienna: '#A0522D', + skyblue: '#87CEEB', + slateblue: '#6A5ACD', + slategray: '#708090', + slategrey: '#708090', + snow: '#FFFAFA', + springgreen: '#00FF7F', + steelblue: '#4682B4', + tan: '#D2B48C', + thistle: '#D8BFD8', + tomato: '#FF6347', + turquoise: '#40E0D0', + violet: '#EE82EE', + wheat: '#F5DEB3', + whitesmoke: '#F5F5F5', + yellowgreen: '#9ACD32', + rebeccapurple: '#663399' + }.stringify_keys.transform_values { Color.new(_1) }.freeze + end + + def self.of(color) + raise ArgumentError, 'No color spec' unless color + return color if color.is_a?(self) + + color = color.to_s.strip + Constants::COLOR_NAME_TO_HEX[color.downcase] || new(color) + end + + def to_s + @value.to_s + end + + def as_json(_options = nil) + to_s + end + + def eql(other) + return false unless other.is_a?(self.class) + + to_s == other.to_s + end + alias_method :==, :eql + + def valid? + PATTERN.match?(@value) + end + + def light? + valid? && rgb.sum > 500 + end + + def luminosity + return :light if light? + + :dark + end + + def contrast + return Constants::DARK if light? + + Constants::LIGHT + end + + private + + def rgb + return [] unless valid? + + @rgb ||= begin + if @value.length == 4 + @value[1, 4].scan(/./).map { |v| (v * 2).hex } + else + @value[1, 7].scan(/.{2}/).map(&:hex) + end + end + end + end +end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index b2bc56f46ee..cc24ae837f3 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -39,6 +39,17 @@ module Gitlab end end + class MutuallyExclusiveKeysValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + mutually_exclusive_keys = value.try(:keys).to_a & options[:in] + + if mutually_exclusive_keys.length > 1 + record.errors.add(attribute, "please use only one the following keys: " + + mutually_exclusive_keys.join(', ')) + end + end + end + class AllowedValuesValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) unless options[:in].include?(value.to_s) @@ -217,12 +228,6 @@ module Gitlab end end - protected - - def fallback - false - end - private def matches_syntax?(value) @@ -231,7 +236,7 @@ module Gitlab def validate_regexp(value) matches_syntax?(value) && - Gitlab::UntrustedRegexp::RubySyntax.valid?(value, fallback: fallback) + Gitlab::UntrustedRegexp::RubySyntax.valid?(value) end end @@ -260,27 +265,6 @@ module Gitlab end end - class ArrayOfStringsOrRegexpsWithFallbackValidator < ArrayOfStringsOrRegexpsValidator - protected - - # TODO - # - # Remove ArrayOfStringsOrRegexpsWithFallbackValidator class too when - # you are removing the `:allow_unsafe_ruby_regexp` feature flag. - # - def validation_message - if ::Feature.enabled?(:allow_unsafe_ruby_regexp, default_enabled: :yaml) - 'should be an array of strings or regular expressions' - else - super - end - end - - def fallback - true - end - end - class ArrayOfStringsOrStringValidator < RegexpValidator def validate_each(record, attribute, value) unless validate_array_of_strings_or_string(value) diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index 78ba0916808..0d4b913b7a0 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -15,7 +15,7 @@ module Gitlab directives = { 'default_src' => "'self'", 'base_uri' => "'self'", - 'connect_src' => "'self'", + 'connect_src' => ContentSecurityPolicy::Directives.connect_src, 'font_src' => "'self'", 'form_action' => "'self' https: http:", 'frame_ancestors' => "'self'", diff --git a/lib/gitlab/content_security_policy/directives.rb b/lib/gitlab/content_security_policy/directives.rb index 3b958f8c92e..4ad420f9e2f 100644 --- a/lib/gitlab/content_security_policy/directives.rb +++ b/lib/gitlab/content_security_policy/directives.rb @@ -7,6 +7,10 @@ module Gitlab module ContentSecurityPolicy module Directives + def self.connect_src + "'self'" + end + def self.frame_src "https://www.google.com/recaptcha/ https://www.recaptcha.net/ https://content.googleapis.com https://content-compute.googleapis.com https://content-cloudbilling.googleapis.com https://content-cloudresourcemanager.googleapis.com https://www.googletagmanager.com/ns.html" end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 0d6767ad564..8ef4977177a 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -70,6 +70,8 @@ module Gitlab else ::ApplicationSetting.create_from_defaults end + rescue ::ApplicationSetting::Recursion + in_memory_application_settings end def fake_application_settings(attributes = {}) diff --git a/lib/gitlab/cycle_analytics/summary/base.rb b/lib/gitlab/cycle_analytics/summary/base.rb index f867dbd4d68..5605d48c543 100644 --- a/lib/gitlab/cycle_analytics/summary/base.rb +++ b/lib/gitlab/cycle_analytics/summary/base.rb @@ -4,27 +4,13 @@ module Gitlab module CycleAnalytics module Summary class Base + include Gitlab::CycleAnalytics::Summary::Defaults + def initialize(project:, options:) @project = project @options = options end - def identifier - self.class.name.demodulize.underscore.to_sym - end - - def title - raise NotImplementedError, "Expected #{self.name} to implement title" - end - - def value - raise NotImplementedError, "Expected #{self.name} to implement value" - end - - def links - [] - end - private attr_reader :project, :options diff --git a/lib/gitlab/cycle_analytics/summary/defaults.rb b/lib/gitlab/cycle_analytics/summary/defaults.rb new file mode 100644 index 00000000000..468494a8ab8 --- /dev/null +++ b/lib/gitlab/cycle_analytics/summary/defaults.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module CycleAnalytics + module Summary + module Defaults + def identifier + self.class.name.demodulize.underscore.to_sym + end + + # :nocov: the class including this concern is expected to test this method. + def title + raise NotImplementedError, "Expected #{self.name} to implement title" + end + # :nocov: + + # :nocov: the class including this concern is expected to test this method. + def value + raise NotImplementedError, "Expected #{self.name} to implement value" + end + # :nocov: + + def links + [] + end + end + end + end +end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 9b32d285ec0..1b16873f737 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -195,6 +195,16 @@ module Gitlab MAX_TIMESTAMP_VALUE > timestamp ? timestamp : MAX_TIMESTAMP_VALUE.dup end + def self.all_uncached(&block) + # Calls to #uncached only disable caching for the current connection. Since the load balancer + # can potentially upgrade from read to read-write mode (using a different connection), we specify + # up-front that we'll explicitly use the primary for the duration of the operation. + Gitlab::Database::LoadBalancing::Session.current.use_primary do + base_models = database_base_models.values + base_models.reduce(block) { |blk, model| -> { model.uncached(&blk) } }.call + end + end + def self.allow_cross_joins_across_databases(url:) # this method is implemented in: # spec/support/database/prevent_cross_joins.rb @@ -221,12 +231,26 @@ module Gitlab ::ActiveRecord::Base.configurations.configs_for(env_name: Rails.env).map(&:name) end + # This returns all matching schemas that a given connection can use + # Since the `ActiveRecord::Base` might change the connection (from main to ci) + # This does not look at literal connection names, but rather compares + # models that are holders for a given db_config_name + def self.gitlab_schemas_for_connection(connection) + connection_name = self.db_config_name(connection) + primary_model = self.database_base_models.fetch(connection_name) + + self.schemas_to_base_models + .select { |_, models| models.include?(primary_model) } + .keys + .map!(&:to_sym) + end + def self.db_config_for_connection(connection) return unless connection - # The LB connection proxy does not have a direct db_config - # that can be referenced - return if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy) + if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy) + return connection.load_balancer.configuration.primary_db_config + end # During application init we might receive `NullPool` return unless connection.respond_to?(:pool) && diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb index 2f990aba2fb..e9846dd4e85 100644 --- a/lib/gitlab/database/async_indexes/migration_helpers.rb +++ b/lib/gitlab/database/async_indexes/migration_helpers.rb @@ -5,6 +5,8 @@ module Gitlab module AsyncIndexes module MigrationHelpers def unprepare_async_index(table_name, column_name, **options) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + return unless async_index_creation_available? index_name = options[:name] || index_name(table_name, column_name) @@ -15,6 +17,8 @@ module Gitlab end def unprepare_async_index_by_name(table_name, index_name, **options) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + return unless async_index_creation_available? PostgresAsyncIndex.find_by(name: index_name).try do |async_index| @@ -32,6 +36,8 @@ module Gitlab # If the requested index has already been created, it is not stored in the table for # asynchronous creation. def prepare_async_index(table_name, column_name, **options) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + return unless async_index_creation_available? index_name = options[:name] || index_name(table_name, column_name) @@ -72,8 +78,7 @@ module Gitlab end def async_index_creation_available? - ApplicationRecord.connection.table_exists?(:postgres_async_indexes) && - Feature.enabled?(:database_async_index_creation, type: :ops) + connection.table_exists?(:postgres_async_indexes) end end end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 185b6d9629f..f3160679d64 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -3,7 +3,9 @@ module Gitlab module Database module BackgroundMigration - class BatchedJob < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + SplitAndRetryError = Class.new(StandardError) + + class BatchedJob < SharedModel include EachBatch include FromUnion @@ -11,6 +13,8 @@ module Gitlab MAX_ATTEMPTS = 3 STUCK_JOBS_TIMEOUT = 1.hour.freeze + TIMEOUT_EXCEPTIONS = [ActiveRecord::StatementTimeout, ActiveRecord::ConnectionTimeoutError, + ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout].freeze belongs_to :batched_migration, foreign_key: :batched_background_migration_id has_many :batched_job_transition_logs, foreign_key: :batched_background_migration_job_id @@ -51,6 +55,16 @@ module Gitlab job.metrics = {} end + after_transition any => :failed do |job, transition| + error_hash = transition.args.find { |arg| arg[:error].present? } + + exception = error_hash&.fetch(:error) + + job.split_and_retry! if job.can_split?(exception) + rescue SplitAndRetryError => error + Gitlab::AppLogger.error(message: error.message, batched_job_id: job.id) + end + after_transition do |job, transition| error_hash = transition.args.find { |arg| arg[:error].present? } @@ -79,20 +93,25 @@ module Gitlab duration.to_f / batched_migration.interval end + def can_split?(exception) + attempts >= MAX_ATTEMPTS && TIMEOUT_EXCEPTIONS.include?(exception&.class) && batch_size > sub_batch_size + end + def split_and_retry! with_lock do - raise 'Only failed jobs can be split' unless failed? + raise SplitAndRetryError, 'Only failed jobs can be split' unless failed? new_batch_size = batch_size / 2 - raise 'Job cannot be split further' if new_batch_size < 1 + raise SplitAndRetryError, 'Job cannot be split further' if new_batch_size < 1 - batching_strategy = batched_migration.batch_class.new + batching_strategy = batched_migration.batch_class.new(connection: self.class.connection) next_batch_bounds = batching_strategy.next_batch( batched_migration.table_name, batched_migration.column_name, batch_min_value: min_value, - batch_size: new_batch_size + batch_size: new_batch_size, + job_arguments: batched_migration.job_arguments ) midpoint = next_batch_bounds.last diff --git a/lib/gitlab/database/background_migration/batched_job_transition_log.rb b/lib/gitlab/database/background_migration/batched_job_transition_log.rb index 418bf1a101f..55a391005a2 100644 --- a/lib/gitlab/database/background_migration/batched_job_transition_log.rb +++ b/lib/gitlab/database/background_migration/batched_job_transition_log.rb @@ -3,7 +3,7 @@ module Gitlab module Database module BackgroundMigration - class BatchedJobTransitionLog < ApplicationRecord + class BatchedJobTransitionLog < SharedModel include PartitionedTable self.table_name = :batched_background_migration_job_transition_logs diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 1f8ca982ed5..65c15795de6 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -3,7 +3,7 @@ module Gitlab module Database module BackgroundMigration - class BatchedMigration < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord + class BatchedMigration < SharedModel JOB_CLASS_MODULE = 'Gitlab::BackgroundMigration' BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies" diff --git a/lib/gitlab/database/background_migration/batched_migration_runner.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 9308bae20cf..06cd40f1e06 100644 --- a/lib/gitlab/database/background_migration/batched_migration_runner.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -6,12 +6,13 @@ module Gitlab class BatchedMigrationRunner FailedToFinalize = Class.new(RuntimeError) - def self.finalize(job_class_name, table_name, column_name, job_arguments) - new.finalize(job_class_name, table_name, column_name, job_arguments) + def self.finalize(job_class_name, table_name, column_name, job_arguments, connection: ApplicationRecord.connection) + new(connection: connection).finalize(job_class_name, table_name, column_name, job_arguments) end - def initialize(migration_wrapper = BatchedMigrationWrapper.new) + def initialize(migration_wrapper = BatchedMigrationWrapper.new, connection: ApplicationRecord.connection) @migration_wrapper = migration_wrapper + @connection = connection end # Runs the next batched_job for a batched_background_migration. @@ -77,7 +78,7 @@ module Gitlab private - attr_reader :migration_wrapper + attr_reader :migration_wrapper, :connection def find_or_create_next_batched_job(active_migration) if next_batch_range = find_next_batch_range(active_migration) @@ -88,7 +89,7 @@ module Gitlab end def find_next_batch_range(active_migration) - batching_strategy = active_migration.batch_class.new + batching_strategy = active_migration.batch_class.new(connection: connection) batch_min_value = active_migration.next_min_value next_batch_bounds = batching_strategy.next_batch( diff --git a/lib/gitlab/database/count/exact_count_strategy.rb b/lib/gitlab/database/count/exact_count_strategy.rb index 0b8fe640bf8..345c7e44b05 100644 --- a/lib/gitlab/database/count/exact_count_strategy.rb +++ b/lib/gitlab/database/count/exact_count_strategy.rb @@ -12,6 +12,7 @@ module Gitlab # Note that for very large tables, this may even timeout. class ExactCountStrategy attr_reader :models + def initialize(models) @models = models end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 68a0c15480a..0f89c500688 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -14,6 +14,7 @@ module Gitlab # however is guaranteed to be "fast", because it only looks up statistics. class ReltuplesCountStrategy attr_reader :models + def initialize(models) @models = models end @@ -46,7 +47,7 @@ module Gitlab end def table_to_model_mapping - @table_to_model_mapping ||= models.each_with_object({}) { |model, h| h[model.table_name] = model } + @table_to_model_mapping ||= models.index_by(&:table_name) end def table_to_model(table_name) diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index c3eea0515d4..cccd4b48723 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -4,8 +4,11 @@ module Gitlab module Database module EachDatabase class << self - def each_database_connection - Gitlab::Database.database_base_models.each_pair do |connection_name, model| + def each_database_connection(only: nil) + selected_names = Array.wrap(only) + base_models = select_base_models(selected_names) + + base_models.each_pair do |connection_name, model| connection = model.connection with_shared_connection(connection, connection_name) do @@ -14,34 +17,52 @@ module Gitlab end end - def each_model_connection(models, &blk) + def each_model_connection(models, only_on: nil, &blk) + selected_databases = Array.wrap(only_on).map(&:to_sym) + models.each do |model| # If model is shared, iterate all available base connections # Example: `LooseForeignKeys::DeletedRecord` if model < ::Gitlab::Database::SharedModel - with_shared_model_connections(model, &blk) + with_shared_model_connections(model, selected_databases, &blk) else - with_model_connection(model, &blk) + with_model_connection(model, selected_databases, &blk) end end end private - def with_shared_model_connections(shared_model, &blk) + def select_base_models(names) + base_models = Gitlab::Database.database_base_models + + return base_models if names.empty? + + names.each_with_object(HashWithIndifferentAccess.new) do |name, hash| + raise ArgumentError, "#{name} is not a valid database name" unless base_models.key?(name) + + hash[name] = base_models[name] + end + end + + def with_shared_model_connections(shared_model, selected_databases, &blk) Gitlab::Database.database_base_models.each_pair do |connection_name, connection_model| if shared_model.limit_connection_names next unless shared_model.limit_connection_names.include?(connection_name.to_sym) end + next if selected_databases.present? && !selected_databases.include?(connection_name.to_sym) + with_shared_connection(connection_model.connection, connection_name) do yield shared_model, connection_name end end end - def with_model_connection(model, &blk) - connection_name = model.connection.pool.db_config.name + def with_model_connection(model, selected_databases, &blk) + connection_name = model.connection_db_config.name + + return if selected_databases.present? && !selected_databases.include?(connection_name.to_sym) with_shared_connection(model.connection, connection_name) do yield model, connection_name diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 7adf6ba6afb..d7ea614e2af 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -95,6 +95,10 @@ module Gitlab def self.tables_to_schema @tables_to_schema ||= YAML.load_file(Rails.root.join('lib/gitlab/database/gitlab_schemas.yml')) end + + def self.schema_names + @schema_names ||= self.tables_to_schema.values.to_set + end end end end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 93cd75ce5a7..dcd78bfd84f 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -8,6 +8,7 @@ alert_management_alert_metric_images: :gitlab_main alert_management_alert_user_mentions: :gitlab_main alert_management_http_integrations: :gitlab_main allowed_email_domains: :gitlab_main +analytics_cycle_analytics_aggregations: :gitlab_main analytics_cycle_analytics_group_stages: :gitlab_main analytics_cycle_analytics_group_value_streams: :gitlab_main analytics_cycle_analytics_issue_stage_events: :gitlab_main @@ -273,6 +274,7 @@ issue_emails: :gitlab_main issue_email_participants: :gitlab_main issue_links: :gitlab_main issue_metrics: :gitlab_main +issue_search_data: :gitlab_main issues: :gitlab_main issues_prometheus_alert_events: :gitlab_main issues_self_managed_prometheus_alert_events: :gitlab_main @@ -379,7 +381,6 @@ pages_deployments: :gitlab_main pages_deployment_states: :gitlab_main pages_domain_acme_orders: :gitlab_main pages_domains: :gitlab_main -partitioned_foreign_keys: :gitlab_main path_locks: :gitlab_main personal_access_tokens: :gitlab_main plan_limits: :gitlab_main @@ -400,6 +401,7 @@ project_alerting_settings: :gitlab_main project_aliases: :gitlab_main project_authorizations: :gitlab_main project_auto_devops: :gitlab_main +project_build_artifacts_size_refreshes: :gitlab_main project_ci_cd_settings: :gitlab_main project_ci_feature_usages: :gitlab_main project_compliance_framework_settings: :gitlab_main @@ -441,6 +443,7 @@ push_event_payloads: :gitlab_main push_rules: :gitlab_main raw_usage_data: :gitlab_main redirect_routes: :gitlab_main +related_epic_links: :gitlab_main release_links: :gitlab_main releases: :gitlab_main remote_mirrors: :gitlab_main @@ -457,6 +460,7 @@ reviews: :gitlab_main routes: :gitlab_main saml_group_links: :gitlab_main saml_providers: :gitlab_main +saved_replies: :gitlab_main schema_migrations: :gitlab_shared scim_identities: :gitlab_main scim_oauth_access_tokens: :gitlab_main diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index e16db5af8ce..6517923d23e 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -47,6 +47,8 @@ module Gitlab # Returns the role (primary/replica) of the database the connection is # connecting to. def self.db_role_for_connection(connection) + return ROLE_UNKNOWN if connection.is_a?(::Gitlab::Database::LoadBalancing::ConnectionProxy) + db_config = Database.db_config_for_connection(connection) return ROLE_UNKNOWN unless db_config diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index 63444ebe169..86b3afaa47b 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -94,6 +94,10 @@ module Gitlab end end + def primary_db_config + primary_model_or_model_if_enabled.connection_db_config + end + def replica_db_config @model.connection_db_config end diff --git a/lib/gitlab/database/load_balancing/setup.rb b/lib/gitlab/database/load_balancing/setup.rb index 126c8bb2aa6..6d667e8ecf0 100644 --- a/lib/gitlab/database/load_balancing/setup.rb +++ b/lib/gitlab/database/load_balancing/setup.rb @@ -90,7 +90,8 @@ module Gitlab def use_model_load_balancing? # Cache environment variable and return env variable first if defined - use_model_load_balancing_env = Gitlab::Utils.to_boolean(ENV["GITLAB_USE_MODEL_LOAD_BALANCING"]) + default_use_model_load_balancing_env = Gitlab.dev_or_test_env? || nil + use_model_load_balancing_env = Gitlab::Utils.to_boolean(ENV.fetch('GITLAB_USE_MODEL_LOAD_BALANCING', default_use_model_load_balancing_env)) unless use_model_load_balancing_env.nil? return use_model_load_balancing_env diff --git a/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb new file mode 100644 index 00000000000..b4e31565c60 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/restrict_gitlab_schema.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module RestrictGitlabSchema + extend ActiveSupport::Concern + + MigrationSkippedError = Class.new(StandardError) + + included do + class_attribute :allowed_gitlab_schemas + end + + class_methods do + def restrict_gitlab_migration(gitlab_schema:) + unless Gitlab::Database::GitlabSchema.schema_names.include?(gitlab_schema) + raise "Unknown 'gitlab_schema: #{gitlab_schema}' specified. It needs to be one of: " \ + "#{Gitlab::Database::GitlabSchema.schema_names.to_a}" + end + + self.allowed_gitlab_schemas = [gitlab_schema] + end + end + + def migrate(direction) + if unmatched_schemas.any? + # TODO: Today skipping migration would raise an exception. + # Ideally, skipped migration should be ignored (not loaded), or softly ignored. + # Read more in: https://gitlab.com/gitlab-org/gitlab/-/issues/355014 + raise MigrationSkippedError, "Current migration is skipped since it modifies "\ + "'#{self.class.allowed_gitlab_schemas}' which is outside of '#{allowed_schemas_for_connection}'" + end + + Gitlab::Database::QueryAnalyzer.instance.within([validator_class]) do + validator_class.allowed_gitlab_schemas = self.allowed_gitlab_schemas + + super + end + end + + private + + def validator_class + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas + end + + def unmatched_schemas + (self.allowed_gitlab_schemas || []) - allowed_schemas_for_connection + end + + def allowed_schemas_for_connection + Gitlab::Database.gitlab_schemas_for_connection(connection) + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 4f1b490cc8f..7e5c002d072 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -42,7 +42,7 @@ module Gitlab # end def queue_background_migration_jobs_by_range_at_intervals(model_class, job_class_name, delay_interval, batch_size: BATCH_SIZE, other_job_arguments: [], initial_delay: 0, track_jobs: false, primary_column_name: :id) raise "#{model_class} does not have an ID column of #{primary_column_name} to use for batch ranges" unless model_class.column_names.include?(primary_column_name.to_s) - raise "#{primary_column_name} is not an integer column" unless model_class.columns_hash[primary_column_name.to_s].type == :integer + raise "#{primary_column_name} is not an integer or string column" unless [:integer, :string].include?(model_class.columns_hash[primary_column_name.to_s].type) job_coordinator = coordinator_for_tracking_database diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 7f34768350b..9d28db6b886 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -17,7 +17,11 @@ module Gitlab def observe(version:, name:, connection:, &block) observation = Observation.new(version: version, name: name, success: false) - observers = observer_classes.map { |c| c.new(observation, @result_dir, connection) } + per_migration_result_dir = File.join(@result_dir, name) + + FileUtils.mkdir_p(per_migration_result_dir) + + observers = observer_classes.map { |c| c.new(observation, per_migration_result_dir, connection) } on_each_observer(observers) { |observer| observer.before } diff --git a/lib/gitlab/database/migrations/observers/query_details.rb b/lib/gitlab/database/migrations/observers/query_details.rb index 8f4406e79a5..1549f5bf626 100644 --- a/lib/gitlab/database/migrations/observers/query_details.rb +++ b/lib/gitlab/database/migrations/observers/query_details.rb @@ -6,7 +6,7 @@ module Gitlab module Observers class QueryDetails < MigrationObserver def before - file_path = File.join(output_dir, "#{observation.version}_#{observation.name}-query-details.json") + file_path = File.join(output_dir, "query-details.json") @file = File.open(file_path, 'wb') @writer = Oj::StreamWriter.new(@file, {}) @writer.push_array diff --git a/lib/gitlab/database/migrations/observers/query_log.rb b/lib/gitlab/database/migrations/observers/query_log.rb index c42fd8bd23d..8ca57bb7df9 100644 --- a/lib/gitlab/database/migrations/observers/query_log.rb +++ b/lib/gitlab/database/migrations/observers/query_log.rb @@ -7,7 +7,7 @@ module Gitlab class QueryLog < MigrationObserver def before @logger_was = ActiveRecord::Base.logger - file_path = File.join(output_dir, "#{observation.version}_#{observation.name}.log") + file_path = File.join(output_dir, "migration.log") @logger = Logger.new(file_path) ActiveRecord::Base.logger = @logger end diff --git a/lib/gitlab/database/migrations/observers/transaction_duration.rb b/lib/gitlab/database/migrations/observers/transaction_duration.rb index a96b94334cf..182aeb10fda 100644 --- a/lib/gitlab/database/migrations/observers/transaction_duration.rb +++ b/lib/gitlab/database/migrations/observers/transaction_duration.rb @@ -6,7 +6,7 @@ module Gitlab module Observers class TransactionDuration < MigrationObserver def before - file_path = File.join(output_dir, "#{observation.version}_#{observation.name}-transaction-duration.json") + file_path = File.join(output_dir, "transaction-duration.json") @file = File.open(file_path, 'wb') @writer = Oj::StreamWriter.new(@file, {}) @writer.push_array diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index f0bac594119..02645a0d452 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -5,6 +5,8 @@ module Gitlab module Migrations class Runner BASE_RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze + METADATA_FILENAME = 'metadata.json' + SCHEMA_VERSION = 2 # Version of the output format produced by the runner class << self def up @@ -75,9 +77,11 @@ module Gitlab end ensure if instrumentation - File.open(File.join(result_dir, Gitlab::Database::Migrations::Instrumentation::STATS_FILENAME), 'wb+') do |io| - io << instrumentation.observations.to_json - end + stats_filename = File.join(result_dir, Gitlab::Database::Migrations::Instrumentation::STATS_FILENAME) + File.write(stats_filename, instrumentation.observations.to_json) + + metadata_filename = File.join(result_dir, METADATA_FILENAME) + File.write(metadata_filename, { version: SCHEMA_VERSION }.to_json) end # We clear the cache here to mirror the cache clearing that happens at the end of `db:migrate` tasks diff --git a/lib/gitlab/database/migrations/test_background_runner.rb b/lib/gitlab/database/migrations/test_background_runner.rb new file mode 100644 index 00000000000..821d68c06c9 --- /dev/null +++ b/lib/gitlab/database/migrations/test_background_runner.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class TestBackgroundRunner + # TODO - build a rake task to call this method, and support it in the gitlab-com-database-testing project. + # Until then, we will inject a migration with a very high timestamp during database testing + # that calls this class to run jobs + # See https://gitlab.com/gitlab-org/database-team/gitlab-com-database-testing/-/issues/41 for details + + def initialize + @job_coordinator = Gitlab::BackgroundMigration.coordinator_for_database(Gitlab::Database::MAIN_DATABASE_NAME) + end + + def traditional_background_migrations + @job_coordinator.pending_jobs + end + + def run_jobs(for_duration:) + jobs_to_run = traditional_background_migrations.group_by { |j| class_name_for_job(j) } + return if jobs_to_run.empty? + + # without .to_f, we do integer division + # For example, 3.minutes / 2 == 1.minute whereas 3.minutes / 2.to_f == (1.minute + 30.seconds) + duration_per_migration_type = for_duration / jobs_to_run.count.to_f + jobs_to_run.each do |_migration_name, jobs| + run_until = duration_per_migration_type.from_now + jobs.shuffle.each do |j| + break if run_until <= Time.current + + run_job(j) + end + end + end + + private + + def run_job(job) + Gitlab::BackgroundMigration.perform(job.args[0], job.args[1]) + end + + def class_name_for_job(job) + job.args[0] + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index c7d8bdf30bc..92825d41599 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -26,10 +26,10 @@ module Gitlab # ignore - happens when Rake tasks yet have to create a database, e.g. for testing end - def sync_partitions(models_to_sync = registered_for_sync) + def sync_partitions(models_to_sync = registered_for_sync, only_on: nil) Gitlab::AppLogger.info(message: 'Syncing dynamic postgres partitions') - Gitlab::Database::EachDatabase.each_model_connection(models_to_sync) do |model| + Gitlab::Database::EachDatabase.each_model_connection(models_to_sync, only_on: only_on) do |model| PartitionManager.new(model).sync_partitions end diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index ab414f91169..3ee9a193b45 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -46,6 +46,7 @@ module Gitlab private attr_reader :model + delegate :connection, to: :model def missing_partitions diff --git a/lib/gitlab/database/partitioning/replace_table.rb b/lib/gitlab/database/partitioning/replace_table.rb index a7686e97553..21a175a660d 100644 --- a/lib/gitlab/database/partitioning/replace_table.rb +++ b/lib/gitlab/database/partitioning/replace_table.rb @@ -31,6 +31,7 @@ module Gitlab private attr_reader :connection + delegate :execute, :quote_table_name, :quote_column_name, to: :connection def default_sequence(table, column) diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb index 2736f9d18dc..0c78dda734c 100644 --- a/lib/gitlab/database/query_analyzer.rb +++ b/lib/gitlab/database/query_analyzer.rb @@ -30,13 +30,17 @@ module Gitlab end end - def within + def within(user_analyzers = nil) # Due to singleton nature of analyzers # only an outer invocation of the `.within` # is allowed to initialize them - return yield if already_within? + if already_within? + raise 'Query analyzers are already defined, cannot re-define them.' if user_analyzers - begin! + return yield + end + + begin!(user_analyzers || all_analyzers) begin yield @@ -61,21 +65,21 @@ module Gitlab next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed) analyzer.analyze(parsed) - rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e + rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e # We catch all standard errors to prevent validation errors to introduce fatal errors in production Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end end # Enable query analyzers - def begin! - analyzers = all_analyzers.select do |analyzer| + def begin!(analyzers = all_analyzers) + analyzers = analyzers.select do |analyzer| if analyzer.enabled? analyzer.begin! true end - rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e + rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) false @@ -88,7 +92,7 @@ module Gitlab def end! enabled_analyzers.select do |analyzer| analyzer.end! - rescue StandardError, QueryAnalyzers::Base::QueryAnalyzerError => e + rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index a604f79dc41..a53da514df2 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -94,7 +94,7 @@ module Gitlab if schemas.many? message = "Cross-database data modification of '#{schemas.to_a.join(", ")}' were detected within " \ - "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables." \ + "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " \ "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception." if schemas.any? { |s| s.to_s.start_with?("undefined") } diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb new file mode 100644 index 00000000000..ab40ba5d59b --- /dev/null +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class RestrictAllowedSchemas < Base + UnsupportedSchemaError = Class.new(QueryAnalyzerError) + DDLNotAllowedError = Class.new(UnsupportedSchemaError) + DMLNotAllowedError = Class.new(UnsupportedSchemaError) + DMLAccessDeniedError = Class.new(UnsupportedSchemaError) + + IGNORED_SCHEMAS = %i[gitlab_shared].freeze + + class << self + def enabled? + true + end + + def allowed_gitlab_schemas + self.context[:allowed_gitlab_schemas] + end + + def allowed_gitlab_schemas=(value) + self.context[:allowed_gitlab_schemas] = value + end + + def analyze(parsed) + # If list of schemas is empty, we allow only DDL changes + if self.dml_mode? + self.restrict_to_dml_only(parsed) + else + self.restrict_to_ddl_only(parsed) + end + end + + def require_ddl_mode!(message = "") + return unless self.context + + self.raise_dml_not_allowed_error(message) if self.dml_mode? + end + + def require_dml_mode!(message = "") + return unless self.context + + self.raise_ddl_not_allowed_error(message) if self.ddl_mode? + end + + private + + def restrict_to_ddl_only(parsed) + tables = self.dml_tables(parsed) + schemas = self.dml_schemas(tables) + + if schemas.any? + self.raise_dml_not_allowed_error("Modifying of '#{tables}' (#{schemas.to_a}) with '#{parsed.sql}'") + end + end + + def restrict_to_dml_only(parsed) + if parsed.pg.ddl_tables.any? + self.raise_ddl_not_allowed_error("Modifying of '#{parsed.pg.ddl_tables}' with '#{parsed.sql}'") + end + + if parsed.pg.ddl_functions.any? + self.raise_ddl_not_allowed_error("Modifying of '#{parsed.pg.ddl_functions}' with '#{parsed.sql}'") + end + + tables = self.dml_tables(parsed) + schemas = self.dml_schemas(tables) + + if (schemas - self.allowed_gitlab_schemas).any? + raise DMLAccessDeniedError, "Select/DML queries (SELECT/UPDATE/DELETE) do access '#{tables}' (#{schemas.to_a}) " \ + "which is outside of list of allowed schemas: '#{self.allowed_gitlab_schemas}'." + end + end + + def dml_mode? + self.allowed_gitlab_schemas&.any? + end + + def ddl_mode? + !self.dml_mode? + end + + def dml_tables(parsed) + parsed.pg.select_tables + parsed.pg.dml_tables + end + + def dml_schemas(tables) + extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables) + extra_schemas.subtract(IGNORED_SCHEMAS) + extra_schemas + end + + def raise_dml_not_allowed_error(message) + raise DMLNotAllowedError, "Select/DML queries (SELECT/UPDATE/DELETE) are disallowed in the DDL (structure) mode. #{message}" + end + + def raise_ddl_not_allowed_error(message) + raise DDLNotAllowedError, "DDL queries (structure) are disallowed in the Select/DML (SELECT/UPDATE/DELETE) mode. #{message}" + end + end + end + end + end +end diff --git a/lib/gitlab/database/transaction/context.rb b/lib/gitlab/database/transaction/context.rb index a902537f02e..6defe9ae443 100644 --- a/lib/gitlab/database/transaction/context.rb +++ b/lib/gitlab/database/transaction/context.rb @@ -6,8 +6,10 @@ module Gitlab class Context attr_reader :context - LOG_SAVEPOINTS_THRESHOLD = 1 # 1 `SAVEPOINT` created in a transaction - LOG_DURATION_S_THRESHOLD = 120 # transaction that is running for 2 minutes or longer + LOG_SAVEPOINTS_THRESHOLD = 1 # 1 `SAVEPOINT` created in a transaction + LOG_DURATION_S_THRESHOLD = 120 # transaction that is running for 2 minutes or longer + LOG_EXTERNAL_HTTP_COUNT_THRESHOLD = 50 # 50 external HTTP requests executed within transaction + LOG_EXTERNAL_HTTP_DURATION_S_THRESHOLD = 1 # 1 second spent in HTTP requests in total within transaction LOG_THROTTLE_DURATION = 1 def initialize @@ -43,6 +45,11 @@ module Gitlab (@context[:backtraces] ||= []).push(cleaned_backtrace) end + def initialize_external_http_tracking + @context[:external_http_count_start] = external_http_requests_count_total + @context[:external_http_duration_start] = external_http_requests_duration_total + end + def duration return unless @context[:start_time].present? @@ -57,10 +64,16 @@ module Gitlab duration.to_i >= LOG_DURATION_S_THRESHOLD end + def external_http_requests_threshold_exceeded? + external_http_requests_count >= LOG_EXTERNAL_HTTP_COUNT_THRESHOLD || + external_http_requests_duration >= LOG_EXTERNAL_HTTP_DURATION_S_THRESHOLD + end + def should_log? return false if logged_already? - savepoints_threshold_exceeded? || duration_threshold_exceeded? + savepoints_threshold_exceeded? || duration_threshold_exceeded? || + external_http_requests_threshold_exceeded? end def commit @@ -75,6 +88,14 @@ module Gitlab @context[:backtraces].to_a end + def external_http_requests_count + @requests_count ||= external_http_requests_count_total - @context[:external_http_count_start].to_i + end + + def external_http_requests_duration + @requests_duration ||= external_http_requests_duration_total - @context[:external_http_duration_start].to_f + end + private def queries @@ -108,6 +129,8 @@ module Gitlab savepoints_count: @context[:savepoints].to_i, rollbacks_count: @context[:rollbacks].to_i, releases_count: @context[:releases].to_i, + external_http_requests_count: external_http_requests_count, + external_http_requests_duration: external_http_requests_duration, sql: queries, savepoint_backtraces: backtraces } @@ -118,6 +141,14 @@ module Gitlab def application_info(attributes) Gitlab::AppJsonLogger.info(attributes) end + + def external_http_requests_count_total + ::Gitlab::Metrics::Subscribers::ExternalHttp.request_count.to_i + end + + def external_http_requests_duration_total + ::Gitlab::Metrics::Subscribers::ExternalHttp.duration.to_f + end end end end diff --git a/lib/gitlab/database/transaction/observer.rb b/lib/gitlab/database/transaction/observer.rb index ad6886a3d52..87e76257c24 100644 --- a/lib/gitlab/database/transaction/observer.rb +++ b/lib/gitlab/database/transaction/observer.rb @@ -21,6 +21,7 @@ module Gitlab context.set_start_time context.set_depth(0) context.track_sql(event.payload[:sql]) + context.initialize_external_http_tracking elsif cmd.start_with?('SAVEPOINT', 'EXCEPTION') context.set_depth(manager.open_transactions) context.increment_savepoints diff --git a/lib/gitlab/database/type/color.rb b/lib/gitlab/database/type/color.rb new file mode 100644 index 00000000000..32ea33a42f3 --- /dev/null +++ b/lib/gitlab/database/type/color.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Type + class Color < ActiveModel::Type::Value + def serialize(value) + value.to_s if value + end + + def serializable?(value) + value.nil? || value.is_a?(::String) || value.is_a?(::Gitlab::Color) + end + + def cast_value(value) + ::Gitlab::Color.new(value.to_s) + end + end + end + end +end diff --git a/lib/gitlab/diff/custom_diff.rb b/lib/gitlab/diff/custom_diff.rb index 3928ece9281..af1fd8fb03e 100644 --- a/lib/gitlab/diff/custom_diff.rb +++ b/lib/gitlab/diff/custom_diff.rb @@ -10,16 +10,16 @@ module Gitlab transformed_for_diff(new_blob, old_blob) Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' }) end - rescue IpynbDiff::InvalidNotebookError => e + rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e Gitlab::ErrorTracking.log_exception(e) nil end def transformed_diff(before, after) transformed_diff = IpynbDiff.diff(before, after, - diff_opts: { context: 5, include_diff_info: true }, - transform_options: { cell_decorator: :percent }, - raise_if_invalid_notebook: true) + raise_if_invalid_nb: true, + diffy_opts: { include_diff_info: true }).to_s(:text) + strip_diff_frontmatter(transformed_diff) end @@ -29,9 +29,7 @@ module Gitlab def transformed_blob_data(blob) if transformed_for_diff?(blob) - IpynbDiff.transform(blob.data, - raise_errors: true, - options: { include_metadata: false, cell_decorator: :percent }) + IpynbDiff.transform(blob.data, raise_errors: true, include_frontmatter: false) end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d9860d9fb86..89822af2455 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -44,11 +44,15 @@ module Gitlab new_blob_lazy old_blob_lazy - diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff if use_custom_diff? + diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff unless use_renderable_diff? end - def use_custom_diff? - strong_memoize(:_custom_diff_enabled) { Feature.enabled?(:jupyter_clean_diffs, repository.project, default_enabled: true) } + def use_renderable_diff? + strong_memoize(:_renderable_diff_enabled) { Feature.enabled?(:rendered_diffs_viewer, repository.project, default_enabled: :yaml) } + end + + def has_renderable? + rendered&.has_renderable? end def position(position_marker, position_type: :text) @@ -292,13 +296,13 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def size - valid_blobs.map(&:size).sum + valid_blobs.sum(&:size) end # rubocop: enable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord def raw_size - valid_blobs.map(&:raw_size).sum + valid_blobs.sum(&:raw_size) end # rubocop: enable CodeReuse/ActiveRecord @@ -370,10 +374,16 @@ module Gitlab lines.none? { |line| line.type.to_s == 'match' } end + def rendered + return unless use_renderable_diff? && ipynb? + + strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) } + end + private def diffable_by_attribute? - repository.attributes(file_path).fetch('diff') { true } + repository.attributes(file_path).fetch('diff', true) end # NOTE: Files with unsupported encodings (e.g. UTF-16) are treated as binary by git, but they are recognized as text files during encoding detection. These files have `Binary files a/filename and b/filename differ' as their raw diff content which cannot be used. We need to handle this special case and avoid displaying incorrect diff. @@ -399,6 +409,10 @@ module Gitlab new_file? || deleted_file? || content_changed? end + def ipynb? + modified_file? && file_path.ends_with?('.ipynb') + end + # We can't use Object#try because Blob doesn't inherit from Object, but # from BasicObject (via SimpleDelegator). def try_blobs(meth) diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index f73e060be7f..7fa1bd6b5ec 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -11,7 +11,7 @@ module Gitlab delegate :count, :size, :real_size, to: :raw_diff_files def self.default_options - ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true) + ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false, include_stats: true, use_extra_viewer_as_main: false) end def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil) @@ -25,6 +25,7 @@ module Gitlab @diff_refs = diff_refs @fallback_diff_refs = fallback_diff_refs @repository = project.repository + @use_extra_viewer_as_main = diff_options.delete(:use_extra_viewer_as_main) end def diffs @@ -120,11 +121,17 @@ module Gitlab stats = diff_stats_collection&.find_by_path(diff.new_path) - Gitlab::Diff::File.new(diff, + diff_file = Gitlab::Diff::File.new(diff, repository: project.repository, diff_refs: diff_refs, fallback_diff_refs: fallback_diff_refs, stats: stats) + + if @use_extra_viewer_as_main && diff_file.has_renderable? + diff_file.rendered + else + diff_file + end end def sort_diffs(diffs) diff --git a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb index b459e3f6619..c6ab56e783a 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff_base.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff_base.rb @@ -12,10 +12,11 @@ module Gitlab @merge_request_diff = merge_request_diff super(merge_request_diff, - project: merge_request_diff.project, - diff_options: diff_options, - diff_refs: merge_request_diff.diff_refs, - fallback_diff_refs: merge_request_diff.fallback_diff_refs) + project: merge_request_diff.project, + diff_options: diff_options, + diff_refs: merge_request_diff.diff_refs, + fallback_diff_refs: merge_request_diff.fallback_diff_refs + ) end def diff_files(sorted: false) diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 6cf414e29cc..c2b834c71b5 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -8,9 +8,9 @@ module Gitlab # SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze - attr_reader :line_code, :marker_ranges - attr_writer :text, :rich_text - attr_accessor :index, :type, :old_pos, :new_pos + attr_reader :marker_ranges + attr_writer :text, :rich_text, :discussable + attr_accessor :index, :type, :old_pos, :new_pos, :line_code def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) @text = text @@ -26,6 +26,7 @@ module Gitlab @line_code = line_code || calculate_line_code @marker_ranges = [] + @discussable = true end def self.init_from_hash(hash) @@ -96,7 +97,7 @@ module Gitlab end def discussable? - !meta? + @discussable && !meta? end def suggestible? diff --git a/lib/gitlab/diff/rendered/notebook/diff_file.rb b/lib/gitlab/diff/rendered/notebook/diff_file.rb new file mode 100644 index 00000000000..e700e730f20 --- /dev/null +++ b/lib/gitlab/diff/rendered/notebook/diff_file.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true +module Gitlab + module Diff + module Rendered + module Notebook + include Gitlab::Utils::StrongMemoize + + class DiffFile < Gitlab::Diff::File + attr_reader :source_diff + + delegate :repository, :diff_refs, :fallback_diff_refs, :unfolded, :unique_identifier, + to: :source_diff + + def initialize(diff_file) + @source_diff = diff_file + end + + def old_blob + return unless notebook_diff + + strong_memoize(:old_blob) { ::Blobs::Notebook.decorate(source_diff.old_blob, notebook_diff.from.as_text) } + end + + def new_blob + return unless notebook_diff + + strong_memoize(:new_blob) { ::Blobs::Notebook.decorate(source_diff.new_blob, notebook_diff.to.as_text) } + end + + def diff + strong_memoize(:diff) { transformed_diff } + end + + def has_renderable? + !notebook_diff.nil? && diff.diff.present? + end + + def rendered + self + end + + def highlighted_diff_lines + @highlighted_diff_lines ||= begin + removal_line_maps, addition_line_maps = compute_end_start_map + Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight.map do |line| + mutate_line(line, addition_line_maps, removal_line_maps) + end + end + end + + private + + def notebook_diff + strong_memoize(:notebook_diff) do + Gitlab::AppLogger.info({ message: 'IPYNB_DIFF_GENERATED' }) + + IpynbDiff.diff(source_diff.old_blob&.data, source_diff.new_blob&.data, + raise_if_invalid_nb: true, + diffy_opts: { include_diff_info: true }) + rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e + Gitlab::ErrorTracking.log_exception(e) + nil + end + end + + def transformed_diff + return unless notebook_diff + + diff = source_diff.diff.clone + diff.diff = strip_diff_frontmatter(notebook_diff.to_s(:text)) + diff + end + + def strip_diff_frontmatter(diff_content) + diff_content.scan(/.*\n/)[2..]&.join('') if diff_content.present? + end + + def transformed_line_to_source(transformed_line, transformed_blocks) + transformed_blocks.empty? ? 0 : ( transformed_blocks[transformed_line - 1][:source_line] || -1 ) + 1 + end + + def mutate_line(line, addition_line_maps, removal_line_maps) + line.new_pos = transformed_line_to_source(line.new_pos, notebook_diff.to.blocks) + line.old_pos = transformed_line_to_source(line.old_pos, notebook_diff.from.blocks) + + line.old_pos = addition_line_maps[line.new_pos] if line.old_pos == 0 && line.new_pos != 0 + line.new_pos = removal_line_maps[line.old_pos] if line.new_pos == 0 && line.old_pos != 0 + + # Lines that do not appear on the original diff should not be commentable + + unless addition_line_maps[line.new_pos] || removal_line_maps[line.old_pos] + line.discussable = false + end + + line.line_code = line_code(line) + line + end + + def compute_end_start_map + # line_codes are used for assigning notes to diffs, and these depend on the line on the new version and the + # line that would have been that one in the previous version. However, since we do a transformation on the + # file, that map gets lost. To overcome this, we look at the original source lines and build two maps: + # - For additions, we look at the latest line change for that line and pick the old line for that id + # - For removals, we look at the first line in the old version, and pick the first line on the new version + # + # + # The caveat here is that we can't have notes on lines that are not a translation of a line in the source + # diff + # + # (gitlab/diff/file.rb:75) + + removals = {} + additions = {} + + source_diff.highlighted_diff_lines.each do |line| + removals[line.old_pos] = line.new_pos + additions[line.new_pos] = line.old_pos + end + + [removals, additions] + end + end + end + end + end +end diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index e213adbfcfd..b67ca8d8a7d 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -15,7 +15,9 @@ module Gitlab filter_signature_attachments(message).each do |attachment| tmp = Tempfile.new("gitlab-email-attachment") begin - File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded } + content = attachment.body.decoded + File.open(tmp.path, "w+b") { |f| f.write content } + sanitize_exif_if_needed(content, tmp.path) file = { tempfile: tmp, @@ -55,6 +57,12 @@ module Gitlab def normalize_mime(content_type) MIME::Type.simplified(content_type, remove_x_prefix: true) end + + # https://gitlab.com/gitlab-org/gitlab/-/issues/239343 + def sanitize_exif_if_needed(content, path) + exif_sanitizer = Gitlab::Sanitizers::Exif.new + exif_sanitizer.clean_existing_path(path, content: content, skip_unallowed_types: true) + end end end end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 71b1d4ed8f9..bb57494c729 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -34,7 +34,7 @@ module Gitlab create_issue_or_note - if from_address + if issue_creator_address add_email_participant send_thank_you_email unless reply_email? end @@ -98,7 +98,7 @@ module Gitlab title: mail.subject, description: message_including_template, confidential: true, - external_author: from_address + external_author: external_author }, spam_params: nil ).execute @@ -176,8 +176,22 @@ module Gitlab ).execute end + def issue_creator_address + reply_to_address || from_address + end + def from_address - (mail.reply_to || []).first || mail.from.first || mail.sender + mail.from.first || mail.sender + end + + def reply_to_address + (mail.reply_to || []).first + end + + def external_author + return issue_creator_address unless reply_to_address && from_address + + _("%{from_address} (reply to: %{reply_to_address})") % { from_address: from_address, reply_to_address: reply_to_address } end def can_handle_legacy_format? @@ -191,7 +205,7 @@ module Gitlab def add_email_participant return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project) - @issue.issue_email_participants.create(email: from_address) + @issue.issue_email_participants.create(email: issue_creator_address) end end end diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb index 77f299bcade..27ba5d2a314 100644 --- a/lib/gitlab/email/html_parser.rb +++ b/lib/gitlab/email/html_parser.rb @@ -8,6 +8,7 @@ module Gitlab end attr_reader :raw_body + def initialize(raw_body) @raw_body = raw_body end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 5b2bbfbe66b..58e7b2f1b44 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -8,6 +8,8 @@ module Gitlab class Receiver include Gitlab::Utils::StrongMemoize + RECEIVED_HEADER_REGEX = /for\s+\<(.+)\>/.freeze + def initialize(raw) @raw = raw end @@ -37,6 +39,8 @@ module Gitlab delivered_to: delivered_to.map(&:value), envelope_to: envelope_to.map(&:value), x_envelope_to: x_envelope_to.map(&:value), + # reduced down to what looks like an email in the received headers + received_recipients: recipients_from_received_headers, meta: { client_id: "email/#{mail.from.first}", project: handler&.project&.full_path @@ -82,7 +86,8 @@ module Gitlab find_key_from_references || find_key_from_delivered_to_header || find_key_from_envelope_to_header || - find_key_from_x_envelope_to_header + find_key_from_x_envelope_to_header || + find_first_key_from_received_headers end def ensure_references_array(references) @@ -117,6 +122,10 @@ module Gitlab Array(mail[:x_envelope_to]) end + def received + Array(mail[:received]) + end + def find_key_from_delivered_to_header delivered_to.find do |header| key = email_class.key_from_address(header.value) @@ -138,6 +147,21 @@ module Gitlab end end + def find_first_key_from_received_headers + return unless ::Feature.enabled?(:use_received_header_for_incoming_emails, default_enabled: :yaml) + + recipients_from_received_headers.find do |email| + key = email_class.key_from_address(email) + break key if key + end + end + + def recipients_from_received_headers + strong_memoize :emails_from_received_headers do + received.map { |header| header.value[RECEIVED_HEADER_REGEX, 1] }.compact + end + end + def ignore_auto_reply! if auto_submitted? || auto_replied? raise AutoGeneratedEmailError diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 6a637306225..259b430a73c 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -23,7 +23,12 @@ module Gitlab ].freeze class << self - def configure + def configure(&block) + configure_raven(&block) + configure_sentry(&block) + end + + def configure_raven Raven.configure do |config| config.dsn = sentry_dsn config.release = Gitlab.revision @@ -34,7 +39,20 @@ module Gitlab # Sanitize authentication headers config.sanitize_http_headers = %w[Authorization Private-Token] - config.before_send = method(:before_send) + config.before_send = method(:before_send_raven) + + yield config if block_given? + end + end + + def configure_sentry + Sentry.init do |config| + config.dsn = new_sentry_dsn + config.release = Gitlab.revision + config.environment = new_sentry_environment + config.before_send = method(:before_send_sentry) + config.background_worker_threads = 0 + config.send_default_pii = true yield config if block_given? end @@ -96,6 +114,18 @@ module Gitlab private + def before_send_raven(event, hint) + return unless Feature.enabled?(:enable_old_sentry_integration, default_enabled: :yaml) + + before_send(event, hint) + end + + def before_send_sentry(event, hint) + return unless Feature.enabled?(:enable_new_sentry_integration, default_enabled: :yaml) + + before_send(event, hint) + end + def before_send(event, hint) inject_context_for_exception(event, hint[:exception]) custom_fingerprinting(event, hint[:exception]) @@ -112,6 +142,13 @@ module Gitlab Raven.capture_exception(exception, **context_payload) end + # There is a possibility that this method is called before Sentry is + # configured. Since Sentry 4.0, some methods of Sentry are forwarded to + # to `nil`, hence we have to check the client as well. + if sentry && ::Sentry.get_current_client && ::Sentry.configuration.dsn + ::Sentry.capture_exception(exception, **context_payload) + end + if logging formatter = Gitlab::ErrorTracking::LogFormatter.new log_hash = formatter.generate_log(exception, context_payload) @@ -121,12 +158,30 @@ module Gitlab end def sentry_dsn - return unless Rails.env.production? || Rails.env.development? + return unless sentry_configurable? return unless Gitlab.config.sentry.enabled Gitlab.config.sentry.dsn end + def new_sentry_dsn + return unless sentry_configurable? + return unless Gitlab::CurrentSettings.respond_to?(:sentry_enabled?) + return unless Gitlab::CurrentSettings.sentry_enabled? + + Gitlab::CurrentSettings.sentry_dsn + end + + def new_sentry_environment + return unless Gitlab::CurrentSettings.respond_to?(:sentry_environment) + + Gitlab::CurrentSettings.sentry_environment + end + + def sentry_configurable? + Rails.env.production? || Rails.env.development? + end + def should_raise_for_dev? Rails.env.development? || Rails.env.test? end diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb index e835deeea2c..045a18f4110 100644 --- a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb +++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb @@ -18,7 +18,7 @@ module Gitlab # only the first one since that's what is used for grouping. def process_first_exception_value(event) # Better in new version, will be event.exception.values - exceptions = event.instance_variable_get(:@interfaces)[:exception]&.values + exceptions = extract_exceptions_from(event) return unless exceptions.is_a?(Array) @@ -37,7 +37,13 @@ module Gitlab # instance variable if message.present? exceptions.each do |exception| - exception.value = message if valid_exception?(exception) + next unless valid_exception?(exception) + + if exception.respond_to?(:value=) + exception.value = message + else + exception.instance_variable_set(:@value, message) + end end end @@ -55,6 +61,14 @@ module Gitlab private + def extract_exceptions_from(event) + if event.is_a?(Raven::Event) + event.instance_variable_get(:@interfaces)[:exception]&.values + else + event.exception&.instance_variable_get(:@values) + end + end + def custom_grpc_fingerprint?(fingerprint) fingerprint.is_a?(Array) && fingerprint.length == 2 && fingerprint[0].start_with?('GRPC::') end @@ -71,7 +85,7 @@ module Gitlab def valid_exception?(exception) case exception - when Raven::SingleExceptionInterface + when Raven::SingleExceptionInterface, Sentry::SingleExceptionInterface exception&.value else false diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index d5bf0cffb1e..a1918ee6ad5 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -67,7 +67,10 @@ module Gitlab add_instrument_for_cache_hit(status_code, route, request) - Gitlab::ApplicationContext.push(feature_category: route.feature_category) + Gitlab::ApplicationContext.push( + feature_category: route.feature_category, + caller_id: route.caller_id + ) new_headers = { 'ETag' => etag, diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index 742b72ecde9..684afc6762a 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -3,22 +3,33 @@ module Gitlab module EtagCaching module Router - Route = Struct.new(:regexp, :name, :feature_category, :router) do + Route = Struct.new(:router, :regexp, :name, :feature_category, :caller_id) do delegate :match, to: :regexp delegate :cache_key, to: :router end module Helpers def build_route(attrs) - EtagCaching::Router::Route.new(*attrs, self) + EtagCaching::Router::Route.new(self, *attrs) + end + + def build_rails_route(attrs) + regexp, name, controller, action_name = *attrs + EtagCaching::Router::Route.new( + self, + regexp, + name, + controller.feature_category_for_action(action_name).to_s, + controller.endpoint_id_for_action(action_name).to_s + ) end end - # Performing RESTful routing match before GraphQL would be more expensive + # Performing Rails routing match before GraphQL would be more expensive # for the GraphQL requests because we need to traverse all of the RESTful # route definitions before falling back to GraphQL. def self.match(request) - Router::Graphql.match(request) || Router::Restful.match(request) + Router::Graphql.match(request) || Router::Rails.match(request) end end end diff --git a/lib/gitlab/etag_caching/router/restful.rb b/lib/gitlab/etag_caching/router/rails.rb index 176676bd6ba..d80c003fe53 100644 --- a/lib/gitlab/etag_caching/router/restful.rb +++ b/lib/gitlab/etag_caching/router/rails.rb @@ -3,7 +3,7 @@ module Gitlab module EtagCaching module Router - class Restful + class Rails extend EtagCaching::Router::Helpers # We enable an ETag for every request matching the regex. @@ -23,74 +23,88 @@ module Gitlab [ %r(#{RESERVED_WORDS_PREFIX}/noteable/issue/\d+/notes\z), 'issue_notes', - 'team_planning' + ::Projects::NotesController, + :index ], [ %r(#{RESERVED_WORDS_PREFIX}/noteable/merge_request/\d+/notes\z), 'merge_request_notes', - 'code_review' + ::Projects::NotesController, + :index ], [ %r(#{RESERVED_WORDS_PREFIX}/issues/\d+/realtime_changes\z), 'issue_title', - 'team_planning' + ::Projects::IssuesController, + :realtime_changes ], [ %r(#{RESERVED_WORDS_PREFIX}/commit/\S+/pipelines\.json\z), 'commit_pipelines', - 'continuous_integration' + ::Projects::CommitController, + :pipelines ], [ %r(#{RESERVED_WORDS_PREFIX}/merge_requests/new\.json\z), 'new_merge_request_pipelines', - 'continuous_integration' + ::Projects::MergeRequests::CreationsController, + :new ], [ %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/pipelines\.json\z), 'merge_request_pipelines', - 'continuous_integration' + ::Projects::MergeRequestsController, + :pipelines ], [ %r(#{RESERVED_WORDS_PREFIX}/pipelines\.json\z), 'project_pipelines', - 'continuous_integration' + ::Projects::PipelinesController, + :index ], [ %r(#{RESERVED_WORDS_PREFIX}/pipelines/\d+\.json\z), 'project_pipeline', - 'continuous_integration' + ::Projects::PipelinesController, + :show ], [ %r(#{RESERVED_WORDS_PREFIX}/builds/\d+\.json\z), 'project_build', - 'continuous_integration' + ::Projects::BuildsController, + :show ], [ %r(#{RESERVED_WORDS_PREFIX}/clusters/\d+/environments\z), 'cluster_environments', - 'continuous_delivery' + ::Groups::ClustersController, + :environments ], [ %r(#{RESERVED_WORDS_PREFIX}/-/environments\.json\z), 'environments', - 'continuous_delivery' + ::Projects::EnvironmentsController, + :index ], [ %r(#{RESERVED_WORDS_PREFIX}/import/github/realtime_changes\.json\z), 'realtime_changes_import_github', - 'importers' + ::Import::GithubController, + :realtime_changes ], [ %r(#{RESERVED_WORDS_PREFIX}/import/gitea/realtime_changes\.json\z), 'realtime_changes_import_gitea', - 'importers' + ::Import::GiteaController, + :realtime_changes ], [ %r(#{RESERVED_WORDS_PREFIX}/merge_requests/\d+/cached_widget\.json\z), 'merge_request_widget', - 'code_review' + ::Projects::MergeRequests::ContentController, + :cached_widget ] - ].map(&method(:build_route)).freeze + ].map(&method(:build_rails_route)).freeze # Overridden in EE to add more routes def self.all_routes @@ -109,4 +123,4 @@ module Gitlab end end -Gitlab::EtagCaching::Router::Restful.prepend_mod_with('Gitlab::EtagCaching::Router::Restful') +Gitlab::EtagCaching::Router::Rails.prepend_mod_with('Gitlab::EtagCaching::Router::Rails') diff --git a/lib/gitlab/experiment/rollout/feature.rb b/lib/gitlab/experiment/rollout/feature.rb index 5a14e3c272e..70c363877b1 100644 --- a/lib/gitlab/experiment/rollout/feature.rb +++ b/lib/gitlab/experiment/rollout/feature.rb @@ -12,10 +12,11 @@ module Gitlab # - not have rolled out the feature flag at all (no percent of actors, # no inclusions, etc.) def enabled? - return false if ::Feature::Definition.get(feature_flag_name).nil? - return false unless Gitlab.dev_env_or_com? + return false unless feature_flag_defined? + return false unless Gitlab.com? + return false unless ::Feature.enabled?(:gitlab_experiment, type: :ops, default_enabled: :yaml) - ::Feature.get(feature_flag_name).state != :off # rubocop:disable Gitlab/AvoidFeatureGet + feature_flag_instance.state != :off end # For assignment we first check to see if our feature flag is enabled @@ -58,6 +59,14 @@ module Gitlab private + def feature_flag_instance + ::Feature.get(feature_flag_name) # rubocop:disable Gitlab/AvoidFeatureGet + end + + def feature_flag_defined? + ::Feature::Definition.get(feature_flag_name).present? + end + def feature_flag_name experiment.name.tr('/', '_') end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 7edda290204..8a5432025d8 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -10,9 +10,9 @@ # The experiment is controlled by a Feature Flag (https://docs.gitlab.com/ee/development/feature_flags/controls.html), # which is named "#{experiment_key}_experiment_percentage" and *must* be set with a percentage and not be used for other purposes. # -# To enable the experiment for 10% of the users: +# To enable the experiment for 10% of the time: # -# chatops: `/chatops run feature set experiment_key_experiment_percentage 10` +# chatops: `/chatops run feature set experiment_key_experiment_percentage 10 --random` # console: `Feature.enable_percentage_of_time(:experiment_key_experiment_percentage, 10)` # # To disable the experiment: diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index 303d952381f..a68e2db4dac 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -20,7 +20,7 @@ module Gitlab end def set_experimentation_subject_id_cookie - if Gitlab.dev_env_or_com? + if Gitlab.com? return if cookies[:experimentation_subject_id].present? cookies.permanent.signed[:experimentation_subject_id] = { diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb index 8ba95520638..b13f55e7969 100644 --- a/lib/gitlab/experimentation/experiment.rb +++ b/lib/gitlab/experimentation/experiment.rb @@ -18,7 +18,7 @@ module Gitlab # Temporary change, we will change `experiment_percentage` in future to `Feature.enabled? Feature.enabled?(feature_flag_name, type: :experiment, default_enabled: :yaml) - ::Gitlab.dev_env_or_com? && experiment_percentage > 0 + ::Gitlab.com? && experiment_percentage > 0 end def enabled_for_index?(index) diff --git a/lib/gitlab/fips.rb b/lib/gitlab/fips.rb new file mode 100644 index 00000000000..1dd363ceb17 --- /dev/null +++ b/lib/gitlab/fips.rb @@ -0,0 +1,25 @@ +# rubocop: disable Naming/FileName +# frozen_string_literal: true + +module Gitlab + class FIPS + # A simple utility class for FIPS-related helpers + + class << self + # Returns whether we should be running in FIPS mode or not + # + # @return [Boolean] + def enabled? + # Attempt to auto-detect FIPS mode from OpenSSL + return true if OpenSSL.fips_mode + + # Otherwise allow it to be set manually via the env vars + return true if ENV["FIPS_MODE"] == "true" + + false + end + end + end +end + +# rubocop: enable Naming/FileName diff --git a/lib/gitlab/form_builders/gitlab_ui_form_builder.rb b/lib/gitlab/form_builders/gitlab_ui_form_builder.rb index 3f9053d4e0c..e8e87a864cc 100644 --- a/lib/gitlab/form_builders/gitlab_ui_form_builder.rb +++ b/lib/gitlab/form_builders/gitlab_ui_form_builder.rb @@ -16,13 +16,15 @@ module Gitlab :div, class: 'gl-form-checkbox custom-control custom-checkbox' ) do + value = checkbox_options[:multiple] ? checked_value : nil + @template.check_box( @object_name, method, format_options(checkbox_options, ['custom-control-input']), checked_value, unchecked_value - ) + generic_label(method, label, label_options, help_text: help_text) + ) + generic_label(method, label, label_options, help_text: help_text, value: value) end end diff --git a/lib/gitlab/front_matter.rb b/lib/gitlab/front_matter.rb index 5c5c74ca1a0..093501e860b 100644 --- a/lib/gitlab/front_matter.rb +++ b/lib/gitlab/front_matter.rb @@ -11,12 +11,12 @@ module Gitlab DELIM = Regexp.union(DELIM_LANG.keys) PATTERN = %r{ - \A(?:[^\r\n]*coding:[^\r\n]*\R)? # optional encoding line - \s* + \A(?<encoding>[^\r\n]*coding:[^\r\n]*\R)? # optional encoding line + (?<before>\s*) ^(?<delim>#{DELIM})[ \t]*(?<lang>\S*)\R # opening front matter marker (optional language specifier) (?<front_matter>.*?) # front matter block content (not greedy) ^(\k<delim> | \.{3}) # closing front matter marker - \s* + [^\S\r\n]*(\R|\z) }mx.freeze end end diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index a5b1b7d914b..5669a65cbd9 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -63,6 +63,7 @@ module Gitlab class BlameLine attr_accessor :lineno, :oldlineno, :commit, :line + def initialize(lineno, oldlineno, commit, line) @lineno = lineno @oldlineno = oldlineno diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index c3ee5b97379..1492ea1ce76 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -99,9 +99,9 @@ module Gitlab gitaly_repository_client.exists? end - def create_repository + def create_repository(default_branch = nil) wrapped_gitaly_errors do - gitaly_repository_client.create_repository + gitaly_repository_client.create_repository(default_branch) rescue GRPC::AlreadyExists => e raise RepositoryExists, e.message end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 194f5da0a5c..4bab94968d7 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -93,9 +93,9 @@ module Gitlab end end - def page(title:, version: nil, dir: nil) + def page(title:, version: nil, dir: nil, load_content: true) wrapped_gitaly_errors do - gitaly_find_page(title: title, version: version, dir: dir) + gitaly_find_page(title: title, version: version, dir: dir, load_content: load_content) end end @@ -121,10 +121,10 @@ module Gitlab gitaly_wiki_client.update_page(page_path, title, format, content, commit_details) end - def gitaly_find_page(title:, version: nil, dir: nil) + def gitaly_find_page(title:, version: nil, dir: nil, load_content: true) return unless title.present? - wiki_page, version = gitaly_wiki_client.find_page(title: title, version: version, dir: dir) + wiki_page, version = gitaly_wiki_client.find_page(title: title, version: version, dir: dir, load_content: load_content) return unless wiki_page Gitlab::Git::WikiPage.new(wiki_page, version) diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index 4d87b91764a..5ae17dbbb91 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -30,10 +30,7 @@ module Gitlab def check(cmd, changes) check_snippet_accessibility! - super.tap do |_| - # Ensure HEAD points to the default branch in case it is not master - snippet.change_head_to_default_branch - end + super end override :download_ability diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index a824f97e197..f376dbce177 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -390,7 +390,7 @@ module Gitlab end def self.long_timeout - if Gitlab::Runtime.web_server? + if Gitlab::Runtime.puma? default_timeout else 6.hours diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index c2b4182f609..0e3f9c2598d 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -212,7 +212,7 @@ module Gitlab ) response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) - response.flat_map(&:stats) + response.flat_map { |rsp| rsp.stats.to_a } end def find_changed_paths(commits) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index adbf07de1b9..4637bf2e3ff 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -119,10 +119,6 @@ module Gitlab response = GitalyClient.call(@repository.storage, :operation_service, :user_merge_to_ref, request, timeout: GitalyClient.long_timeout) - if pre_receive_error = response.pre_receive_error.presence - raise Gitlab::Git::PreReceiveError, pre_receive_error - end - response.commit_id end @@ -153,10 +149,6 @@ module Gitlab second_response = response_enum.next - if second_response.pre_receive_error.present? - raise Gitlab::Git::PreReceiveError, second_response.pre_receive_error - end - branch_update = second_response.branch_update return if branch_update.nil? raise Gitlab::Git::CommitError, 'failed to apply merge to branch' unless branch_update.commit_id.present? @@ -164,16 +156,20 @@ module Gitlab Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) rescue GRPC::BadStatus => e - decoded_error = decode_detailed_error(e) - - raise unless decoded_error.present? - - # We simply ignore any reference update errors which are typically an - # indicator of multiple RPC calls trying to update the same reference - # at the same point in time. - return if decoded_error.is_a?(Gitlab::Git::ReferenceUpdateError) + detailed_error = decode_detailed_error(e) - raise decoded_error + case detailed_error&.error + when :access_check + access_check_error = detailed_error.access_check + # These messages were returned from internal/allowed API calls + raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) + when :reference_update + # We simply ignore any reference update errors which are typically an + # indicator of multiple RPC calls trying to update the same reference + # at the same point in time. + else + raise + end ensure request_enum.close end @@ -267,6 +263,19 @@ module Gitlab perform_next_gitaly_rebase_request(response_enum) rebase_sha + rescue GRPC::BadStatus => e + detailed_error = decode_detailed_error(e) + + case detailed_error&.error + when :access_check + access_check_error = detailed_error.access_check + # These messages were returned from internal/allowed API calls + raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) + when :rebase_conflict + raise Gitlab::Git::Repository::GitError, e.details + else + raise e + end ensure request_enum.close end @@ -295,6 +304,26 @@ module Gitlab end response.squash_sha + rescue GRPC::BadStatus => e + detailed_error = decode_detailed_error(e) + + case detailed_error&.error + when :resolve_revision, :rebase_conflict + # Theoretically, we could now raise specific errors based on the type + # of the detailed error. Most importantly, we get error details when + # Gitaly was not able to resolve the `start_sha` or `end_sha` via a + # ResolveRevisionError, and we get information about which files are + # conflicting via a MergeConflictError. + # + # We don't do this now though such that we can maintain backwards + # compatibility with the minimum required set of changes during the + # transitory period where we're migrating UserSquash to use + # structured errors. We thus continue to just return a GitError, like + # we previously did. + raise Gitlab::Git::Repository::GitError, e.details + else + raise + end end def user_update_submodule(user:, submodule:, commit_sha:, branch:, message:) @@ -492,23 +521,7 @@ module Gitlab prefix = %r{type\.googleapis\.com\/gitaly\.(?<error_type>.+)} error_type = prefix.match(detailed_error.type_url)[:error_type] - detailed_error = Gitaly.const_get(error_type, false).decode(detailed_error.value) - - case detailed_error.error - when :access_check - access_check_error = detailed_error.access_check - # These messages were returned from internal/allowed API calls - Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) - when :reference_update - reference_update_error = detailed_error.reference_update - Gitlab::Git::ReferenceUpdateError.new(err.details, - reference_update_error.reference_name, - reference_update_error.old_oid, - reference_update_error.new_oid) - else - # We're handling access_check only for now, but we'll add more detailed error types - nil - end + Gitaly.const_get(error_type, false).decode(detailed_error.value) rescue NameError, NoMethodError # Error Class might not be known to ruby yet nil diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 73848dfff5d..5c447dfd417 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -21,6 +21,16 @@ module Gitlab response.exists end + def optimize_repository + request = Gitaly::OptimizeRepositoryRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :optimize_repository, request, timeout: GitalyClient.long_timeout) + end + + def prune_unreachable_objects + request = Gitaly::PruneUnreachableObjectsRequest.new(repository: @gitaly_repo) + GitalyClient.call(@storage, :repository_service, :prune_unreachable_objects, request, timeout: GitalyClient.long_timeout) + end + def garbage_collect(create_bitmap, prune:) request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap, prune: prune) GitalyClient.call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout) @@ -97,8 +107,8 @@ module Gitlab end # rubocop: enable Metrics/ParameterLists - def create_repository - request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo) + def create_repository(default_branch = nil) + request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: default_branch) GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 3613cd01122..ca839b232cf 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -64,12 +64,13 @@ module Gitlab GitalyClient.call(@repository.storage, :wiki_service, :wiki_update_page, enum, timeout: GitalyClient.medium_timeout) end - def find_page(title:, version: nil, dir: nil) + def find_page(title:, version: nil, dir: nil, load_content: true) request = Gitaly::WikiFindPageRequest.new( repository: @gitaly_repo, title: encode_binary(title), revision: encode_binary(version), - directory: encode_binary(dir) + directory: encode_binary(dir), + skip_content: !load_content ) response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request, timeout: GitalyClient.fast_timeout) diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index 02b582190b6..a9f8483d8c3 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -4,6 +4,8 @@ module Gitlab module GithubImport module Importer class DiffNoteImporter + DiffNoteCreationError = Class.new(ActiveRecord::RecordInvalid) + # note - An instance of `Gitlab::GithubImport::Representation::DiffNote` # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` @@ -31,7 +33,7 @@ module Gitlab else import_with_legacy_diff_note end - rescue ::DiffNote::NoteDiffFileCreationError => e + rescue ::DiffNote::NoteDiffFileCreationError, DiffNoteCreationError => e Logger.warn(message: e.message, 'error.class': e.class.name) import_with_legacy_diff_note @@ -84,7 +86,7 @@ module Gitlab def import_with_diff_note log_diff_note_creation('DiffNote') - ::Import::Github::Notes::CreateService.new(project, author, { + record = ::Import::Github::Notes::CreateService.new(project, author, { noteable_type: note.noteable_type, system: false, type: 'DiffNote', @@ -97,6 +99,8 @@ module Gitlab updated_at: note.updated_at, position: note.diff_position }).execute + + raise DiffNoteCreationError, record unless record.persisted? end def note_body diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index fc0c099b71c..5d291d9d723 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -74,6 +74,10 @@ module Gitlab { state: 'all', sort: 'created', direction: 'asc' } end + def parallel_import_batch + { size: 200, delay: 1.minute } + end + def repository_updates_counter @repository_updates_counter ||= Gitlab::Metrics.counter( :github_importer_repository_updates, diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index a8e006ea082..4dec9543a13 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -72,6 +72,14 @@ module Gitlab # Imports all objects in parallel by scheduling a Sidekiq job for every # individual object. def parallel_import + if Feature.enabled?(:spread_parallel_import, default_enabled: :yaml) && parallel_import_batch.present? + spread_parallel_import + else + parallel_import_deprecated + end + end + + def parallel_import_deprecated waiter = JobWaiter.new each_object_to_import do |object| @@ -86,6 +94,33 @@ module Gitlab waiter end + def spread_parallel_import + waiter = JobWaiter.new + + import_arguments = [] + + each_object_to_import do |object| + repr = representation_class.from_api_response(object) + + import_arguments << [project.id, repr.to_hash, waiter.key] + + waiter.jobs_remaining += 1 + end + + # rubocop:disable Scalability/BulkPerformWithContext + Gitlab::ApplicationContext.with_context(project: project) do + sidekiq_worker_class.bulk_perform_in( + 1.second, + import_arguments, + batch_size: parallel_import_batch[:size], + batch_delay: parallel_import_batch[:delay] + ) + end + # rubocop:enable Scalability/BulkPerformWithContext + + waiter + end + # The method that will be called for traversing through all the objects to # import, yielding them to the supplied block. def each_object_to_import @@ -171,6 +206,12 @@ module Gitlab raise NotImplementedError end + # Default batch settings for parallel import (can be redefined in Importer classes) + # Example: { size: 100, delay: 1.minute } + def parallel_import_batch + {} + end + def abort_on_failure false end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 2bd59415771..9f18513f066 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -40,7 +40,6 @@ module Gitlab gon.ee = Gitlab.ee? gon.jh = Gitlab.jh? gon.dot_com = Gitlab.com? - gon.dev_env_or_com = Gitlab.dev_env_or_com? if current_user gon.current_user_id = current_user.id @@ -52,13 +51,15 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend - push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) push_frontend_feature_flag(:new_header_search, default_enabled: :yaml) push_frontend_feature_flag(:bootstrap_confirmation_modals, default_enabled: :yaml) push_frontend_feature_flag(:sandboxed_mermaid, default_enabled: :yaml) + push_frontend_feature_flag(:source_editor_toolbar, default_enabled: :yaml) + push_frontend_feature_flag(:gl_avatar_for_all_user_avatars, default_enabled: :yaml) + push_frontend_feature_flag(:mr_attention_requests, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/batch_key.rb b/lib/gitlab/graphql/batch_key.rb index 51203af5a43..553e0573c63 100644 --- a/lib/gitlab/graphql/batch_key.rb +++ b/lib/gitlab/graphql/batch_key.rb @@ -4,6 +4,7 @@ module Gitlab module Graphql class BatchKey attr_reader :object + delegate :hash, to: :object def initialize(object, lookahead = nil, object_name: nil) diff --git a/lib/gitlab/graphql/loaders/batch_commit_loader.rb b/lib/gitlab/graphql/loaders/batch_commit_loader.rb new file mode 100644 index 00000000000..26c1f61c567 --- /dev/null +++ b/lib/gitlab/graphql/loaders/batch_commit_loader.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + class BatchCommitLoader + def initialize(container_class:, container_id:, oid:) + @container_class = container_class + @container_id = container_id + @oid = oid + end + + def find + Gitlab::Graphql::Lazy.with_value(find_containers) do |container| + BatchLoader::GraphQL.for(oid).batch(key: container) do |oids, loader, args| + container = args[:key] + + container.repository.commits_by(oids: oids).each do |commit| + loader.call(commit.id, commit) if commit + end + end + end + end + + private + + def find_containers + BatchLoader::GraphQL.for(container_id.to_i).batch(key: container_class) do |ids, loader, args| + model = args[:key] + results = model.includes(:route).id_in(ids) # rubocop: disable CodeReuse/ActiveRecord + + results.each { |record| loader.call(record.id, record) } + end + end + + attr_reader :container_class, :container_id, :oid + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb index 15f95edd318..e8335a3c79c 100644 --- a/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb +++ b/lib/gitlab/graphql/pagination/keyset/generic_keyset_pagination.rb @@ -17,21 +17,13 @@ module Gitlab strong_memoize(:generic_keyset_pagination_has_next_page) do if before - # If `before` is specified, that points to a specific record, - # even if it's the last one. Since we're asking for `before`, - # then the specific record we're pointing to is in the - # next page true elsif first case sliced_nodes when Array sliced_nodes.size > limit_value else - # If we count the number of requested items plus one (`limit_value + 1`), - # then if we get `limit_value + 1` then we know there is a next page sliced_nodes.limit(1).offset(limit_value).exists? - # replacing relation count - # relation_count(set_limit(sliced_nodes, limit_value + 1)) == limit_value + 1 end else false diff --git a/lib/gitlab/harbor/client.rb b/lib/gitlab/harbor/client.rb new file mode 100644 index 00000000000..06142ae2b40 --- /dev/null +++ b/lib/gitlab/harbor/client.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Harbor + class Client + Error = Class.new(StandardError) + ConfigError = Class.new(Error) + + attr_reader :integration + + def initialize(integration) + raise ConfigError, 'Please check your integration configuration.' unless integration + + @integration = integration + end + + def ping + options = { headers: headers.merge!('Accept': 'text/plain') } + response = Gitlab::HTTP.get(url('ping'), options) + + { success: response.success? } + end + + private + + def url(path) + Gitlab::Utils.append_path(base_url, path) + end + + def base_url + Gitlab::Utils.append_path(integration.url, '/api/v2.0/') + end + + def headers + auth = Base64.strict_encode64("#{integration.username}:#{integration.password}") + { + 'Content-Type': 'application/json', + 'Authorization': "Basic #{auth}" + } + end + end + end +end diff --git a/lib/gitlab/health_checks/db_check.rb b/lib/gitlab/health_checks/db_check.rb index ec4b97eaca4..3df312af1bc 100644 --- a/lib/gitlab/health_checks/db_check.rb +++ b/lib/gitlab/health_checks/db_check.rb @@ -13,12 +13,14 @@ module Gitlab end def successful?(result) - result == '1' + result == Gitlab::Database.database_base_models.size end def check catch_timeout 10.seconds do - ActiveRecord::Base.connection.execute('SELECT 1 as ping')&.first&.[]('ping')&.to_s + Gitlab::Database.database_base_models.sum do |_, base| + base.connection.select_value('SELECT 1') + end end end end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 49712548960..758a594036b 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -11,11 +11,7 @@ module Gitlab end def self.too_large?(size) - return false unless size.to_i > self.file_size_limit - - over_highlight_size_limit.increment(source: "file size: #{self.file_size_limit}") if Feature.enabled?(:track_file_size_over_highlight_limit) - - true + size.to_i > self.file_size_limit end attr_reader :blob_name @@ -74,14 +70,10 @@ module Gitlab end def highlight_rich(text, continue: true) - add_highlight_attempt_metric - tag = lexer.tag tokens = lexer.lex(text, continue: continue) Timeout.timeout(timeout_time) { @formatter.format(tokens, **context, tag: tag).html_safe } rescue Timeout::Error => e - add_highlight_timeout_metric - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) highlight_plain(text) rescue StandardError @@ -95,38 +87,5 @@ module Gitlab def link_dependencies(text, highlighted_text) Gitlab::DependencyLinker.link(blob_name, text, highlighted_text) end - - def add_highlight_attempt_metric - return unless Feature.enabled?(:track_highlight_timeouts) - - highlighting_attempt.increment(source: (@language || "undefined")) - end - - def add_highlight_timeout_metric - return unless Feature.enabled?(:track_highlight_timeouts) - - highlight_timeout.increment(source: Gitlab::Runtime.sidekiq? ? "background" : "foreground") - end - - def highlighting_attempt - @highlight_attempt ||= Gitlab::Metrics.counter( - :file_highlighting_attempt, - 'Counts the times highlighting has been attempted on a file' - ) - end - - def highlight_timeout - @highlight_timeout ||= Gitlab::Metrics.counter( - :highlight_timeout, - 'Counts the times highlights have timed out' - ) - end - - def self.over_highlight_size_limit - @over_highlight_size_limit ||= Gitlab::Metrics.counter( - :over_highlight_size_limit, - 'Count the times files have been over the highlight size limit' - ) - end end end diff --git a/lib/gitlab/hook_data/issuable_builder.rb b/lib/gitlab/hook_data/issuable_builder.rb index b8da6731081..5c8aa5050ed 100644 --- a/lib/gitlab/hook_data/issuable_builder.rb +++ b/lib/gitlab/hook_data/issuable_builder.rb @@ -26,7 +26,7 @@ module Gitlab end def safe_keys - issuable_builder.safe_hook_attributes + issuable_builder::SAFE_HOOK_RELATIONS + issuable_builder.safe_hook_attributes + issuable_builder.safe_hook_relations end private diff --git a/lib/gitlab/hook_data/issue_builder.rb b/lib/gitlab/hook_data/issue_builder.rb index 181ce447b52..bd0603c5e5b 100644 --- a/lib/gitlab/hook_data/issue_builder.rb +++ b/lib/gitlab/hook_data/issue_builder.rb @@ -3,13 +3,16 @@ module Gitlab module HookData class IssueBuilder < BaseBuilder - SAFE_HOOK_RELATIONS = %i[ - assignees - labels - total_time_spent - time_change - severity - ].freeze + def self.safe_hook_relations + %i[ + assignees + labels + total_time_spent + time_change + severity + escalation_status + ].freeze + end def self.safe_hook_attributes %i[ @@ -56,6 +59,10 @@ module Gitlab severity: issue.severity } + if issue.supports_escalation? && issue.escalation_status + attrs[:escalation_status] = issue.escalation_status.status_name + end + issue.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) .merge!(attrs) end diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index 0e787a77a25..aaca16d8d7c 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -34,12 +34,14 @@ module Gitlab ].freeze end - SAFE_HOOK_RELATIONS = %i[ - assignees - labels - total_time_spent - time_change - ].freeze + def self.safe_hook_relations + %i[ + assignees + labels + total_time_spent + time_change + ].freeze + end alias_method :merge_request, :object diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index dfecf3a669e..002708beb3c 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -29,7 +29,7 @@ module Gitlab http = super http.hostname_override = hostname if hostname - if Feature.enabled?(:header_read_timeout_buffered_io) + if Feature.enabled?(:header_read_timeout_buffered_io, default_enabled: :yaml) gitlab_http = Gitlab::NetHttpAdapter.new(http.address, http.port) http.instance_variables.each do |variable| @@ -47,6 +47,7 @@ module Gitlab def validate_url!(url) Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?, + allow_object_storage: allow_object_storage?, dns_rebind_protection: dns_rebind_protection?) rescue Gitlab::UrlBlocker::BlockedUrlError => e raise Gitlab::HTTP::BlockedUrlError, "URL '#{url}' is blocked: #{e.message}" @@ -56,6 +57,10 @@ module Gitlab options.fetch(:allow_local_requests, allow_settings_local_requests?) end + def allow_object_storage? + options.fetch(:allow_object_storage, false) + end + def dns_rebind_protection? return false if Gitlab.http_proxy_env? diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 584f7d4aeaf..d01f7d0074f 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -43,27 +43,27 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 48, + 'da_DK' => 46, 'de' => 15, 'en' => 100, 'eo' => 0, - 'es' => 39, + 'es' => 40, 'fil_PH' => 0, 'fr' => 11, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 2, - 'ja' => 35, - 'ko' => 13, - 'nb_NO' => 31, + 'ja' => 34, + 'ko' => 12, + 'nb_NO' => 30, 'nl_NL' => 0, 'pl_PL' => 4, - 'pt_BR' => 50, + 'pt_BR' => 49, 'ro_RO' => 22, 'ru' => 32, 'tr_TR' => 14, - 'uk' => 44, - 'zh_CN' => 96, + 'uk' => 48, + 'zh_CN' => 95, 'zh_HK' => 2, 'zh_TW' => 2 }.freeze diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 8a8c74c302d..53dd6f8cd55 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -300,7 +300,7 @@ module Gitlab return cache[table_name] if cache.has_key?(table_name) index_exists = - ActiveRecord::Base.connection.index_exists?( + relation_class.connection.index_exists?( relation_class.table_name, importable_foreign_key, unique: true) diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb new file mode 100644 index 00000000000..d0fae2cbb95 --- /dev/null +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +# RelationObjectSaver allows for an alternative approach to persisting +# objects during Project/Group Import which persists object's +# nested collection subrelations separately, in batches. +# +# Instead of the regular `relation_object.save!` that opens one db +# transaction for the object itself and all of its subrelations we +# separate collection subrelations from the object and save them +# in batches in smaller more frequent db transactions. +module Gitlab + module ImportExport + module Base + class RelationObjectSaver + include Gitlab::Utils::StrongMemoize + + BATCH_SIZE = 100 + MIN_RECORDS_SIZE = 5 + + # @param relation_object [Object] Object of a project/group, e.g. an issue + # @param relation_key [String] Name of the object association to group/project, e.g. :issues + # @param relation_definition [Hash] Object subrelations as defined in import_export.yml + # @param importable [Project|Group] Project or group where relation object is getting saved to + # + # @example + # Gitlab::ImportExport::Base::RelationObjectSaver.new( + # relation_key: 'merge_requests', + # relation_object: #<MergeRequest id: root/mrs!1, notes: [#<Note id: nil, note: 'test', ...>, #<Note id: nil, noteL 'another note'>]>, + # relation_definition: {"metrics"=>{}, "award_emoji"=>{}, "notes"=>{"author"=>{}, ... }} + # importable: @importable + # ).execute + def initialize(relation_object:, relation_key:, relation_definition:, importable:) + @relation_object = relation_object + @relation_key = relation_key + @relation_definition = relation_definition + @importable = importable + @invalid_subrelations = [] + end + + def execute + move_subrelations + + relation_object.save! + + save_subrelations + ensure + log_invalid_subrelations + end + + private + + attr_reader :relation_object, :relation_key, :relation_definition, + :importable, :collection_subrelations, :invalid_subrelations + + # rubocop:disable GitlabSecurity/PublicSend + def save_subrelations + collection_subrelations.each_pair do |relation_name, records| + records.each_slice(BATCH_SIZE) do |batch| + valid_records, invalid_records = batch.partition { |record| record.valid? } + + invalid_subrelations << invalid_records + relation_object.public_send(relation_name) << valid_records + end + end + end + + def move_subrelations + strong_memoize(:collection_subrelations) do + relation_definition.each_key.each_with_object({}) do |definition, collection_subrelations| + subrelation = relation_object.public_send(definition) + association = relation_object.class.reflect_on_association(definition) + + if association&.collection? && subrelation.size > MIN_RECORDS_SIZE + collection_subrelations[definition] = subrelation.records + + subrelation.clear + end + end + end + end + # rubocop:enable GitlabSecurity/PublicSend + + def log_invalid_subrelations + invalid_subrelations.flatten.each do |record| + Gitlab::Import::Logger.info( + message: '[Project/Group Import] Invalid subrelation', + importable_column_name => importable.id, + relation_key: relation_key, + error_messages: record.errors.full_messages.to_sentence + ) + + ImportFailure.create( + source: 'RelationObjectSaver#save!', + relation_key: relation_key, + exception_class: 'RecordInvalid', + exception_message: record.errors.full_messages.to_sentence, + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, + importable_column_name => importable.id + ) + end + end + + def importable_column_name + @column_name ||= importable.class.reflect_on_association(:import_failures).foreign_key.to_sym + end + end + end + end +end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index e520cade517..2b0467d8779 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -6,6 +6,8 @@ module Gitlab UNTAR_MASK = 'u+rwX,go+rX,go-w' DEFAULT_DIR_MODE = 0700 + FileOversizedError = Class.new(StandardError) + def tar_czf(archive:, dir:) tar_with_options(archive: archive, dir: dir, options: 'czf') end @@ -51,19 +53,34 @@ module Gitlab private - def download_or_copy_upload(uploader, upload_path) + def download_or_copy_upload(uploader, upload_path, size_limit: nil) if uploader.upload.local? copy_files(uploader.path, upload_path) else - download(uploader.url, upload_path) + download(uploader.url, upload_path, size_limit: size_limit) end end - def download(url, upload_path) - File.open(upload_path, 'w') do |file| - # Download (stream) file from the uploader's location - IO.copy_stream(URI.parse(url).open, file) + def download(url, upload_path, size_limit: nil) + File.open(upload_path, 'wb') do |file| + current_size = 0 + + Gitlab::HTTP.get(url, stream_body: true, allow_object_storage: true) do |fragment| + if [301, 302, 307].include?(fragment.code) + Gitlab::Import::Logger.warn(message: "received redirect fragment", fragment_code: fragment.code) + elsif fragment.code == 200 + current_size += fragment.bytesize + + raise FileOversizedError if size_limit.present? && current_size > size_limit + + file.write(fragment) + else + raise Gitlab::ImportExport::Error, "unsupported response downloading fragment #{fragment.code}" + end + end end + rescue FileOversizedError + nil end def tar_with_options(archive:, dir:, options:) diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 5274fcec43e..829b3771518 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -72,9 +72,17 @@ module Gitlab import_export_upload = @importable.import_export_upload if import_export_upload.remote_import_url.present? - download(import_export_upload.remote_import_url, @archive_file) + download( + import_export_upload.remote_import_url, + @archive_file, + size_limit: ::Import::GitlabProjects::RemoteFileValidator::FILE_SIZE_LIMIT + ) else - download_or_copy_upload(import_export_upload.import_file, @archive_file) + download_or_copy_upload( + import_export_upload.import_file, + @archive_file, + size_limit: ::Import::GitlabProjects::RemoteFileValidator::FILE_SIZE_LIMIT + ) end end diff --git a/lib/gitlab/import_export/group/object_builder.rb b/lib/gitlab/import_export/group/object_builder.rb index 43cc7a78a61..e26f37c3347 100644 --- a/lib/gitlab/import_export/group/object_builder.rb +++ b/lib/gitlab/import_export/group/object_builder.rb @@ -13,21 +13,12 @@ module Gitlab super @group = @attributes['group'] - - update_description end private attr_reader :group - # Convert description empty string to nil - # due to existing object being saved with description: nil - # Which makes object lookup to fail since nil != '' - def update_description - attributes['description'] = nil if attributes['description'] == '' - end - def where_clauses [ where_clause_base, diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index c2cbd2fdf47..b44874f598c 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -29,7 +29,7 @@ module Gitlab end def restore - ActiveRecord::Base.uncached do + Gitlab::Database.all_uncached do ActiveRecord::Base.no_touching do update_params! @@ -79,10 +79,7 @@ module Gitlab relation_object.assign_attributes(importable_class_sym => @importable) - import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do - relation_object.save! - log_relation_creation(@importable, relation_key, relation_object) - end + save_relation_object(relation_object, relation_key, relation_definition, relation_index) rescue StandardError => e import_failure_service.log_import_failure( source: 'process_relation_item!', @@ -91,6 +88,23 @@ module Gitlab exception: e) end + def save_relation_object(relation_object, relation_key, relation_definition, relation_index) + if Feature.enabled?(:import_relation_object_persistence, default_enabled: :yaml) && relation_object.new_record? + Gitlab::ImportExport::Base::RelationObjectSaver.new( + relation_object: relation_object, + relation_key: relation_key, + relation_definition: relation_definition, + importable: @importable + ).execute + else + import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do + relation_object.save! + end + end + + log_relation_creation(@importable, relation_key, relation_object) + end + def import_failure_service @import_failure_service ||= ImportFailureService.new(@importable) end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index d893c8dfaa3..55b8c1d4531 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -166,8 +166,6 @@ module Gitlab end def read_from_replica_if_available(&block) - return yield unless ::Feature.enabled?(:load_balancing_for_export_workers, type: :development, default_enabled: :yaml) - ::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries(&block) end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 059f6bd42e3..fc05cc1a79c 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -370,6 +370,7 @@ included_attributes: - :name - :email events: + - :project_id - :target_type - :action - :author_id diff --git a/lib/gitlab/insecure_key_fingerprint.rb b/lib/gitlab/insecure_key_fingerprint.rb index 7b1cf5e7931..ef342f3819f 100644 --- a/lib/gitlab/insecure_key_fingerprint.rb +++ b/lib/gitlab/insecure_key_fingerprint.rb @@ -10,6 +10,7 @@ module Gitlab # class InsecureKeyFingerprint attr_accessor :key + alias_attribute :fingerprint_md5, :fingerprint # diff --git a/lib/gitlab/integrations/sti_type.rb b/lib/gitlab/integrations/sti_type.rb index 1350d75b216..82c2b3297c1 100644 --- a/lib/gitlab/integrations/sti_type.rb +++ b/lib/gitlab/integrations/sti_type.rb @@ -5,7 +5,7 @@ module Gitlab class StiType < ActiveRecord::Type::String NAMESPACED_INTEGRATIONS = Set.new(%w( Asana Assembla Bamboo Bugzilla Buildkite Campfire Confluence CustomIssueTracker Datadog - Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Irker Jenkins Jira Mattermost + Discord DroneCi EmailsOnPush Ewm ExternalWiki Flowdock HangoutsChat Harbor Irker Jenkins Jira Mattermost MattermostSlashCommands MicrosoftTeams MockCi MockMonitoring Packagist PipelinesEmail Pivotaltracker Prometheus Pushover Redmine Shimo Slack SlackSlashCommands Teamcity UnifyCircuit WebexTeams Youtrack Zentao )).freeze diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 368b621bdfb..9824b46554f 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -16,6 +16,9 @@ module Gitlab # @return [Boolean, String, Array, Hash] # @raise [JSON::ParserError] raised if parsing fails def parse(string, opts = {}) + # Parse nil as nil + return if string.nil? + # First we should ensure this really is a string, not some other # type which purports to be a string. This handles some legacy # usage of the JSON class. @@ -30,6 +33,7 @@ module Gitlab end alias_method :parse!, :parse + alias_method :load, :parse # Restricted method for converting a Ruby object to JSON. If you # need to pass options to this, you should use `.generate` instead, @@ -67,6 +71,14 @@ module Gitlab ::JSON.pretty_generate(object, opts) end + # The standard parser error we should be returning. Defined in a method + # so we can potentially override it later. + # + # @return [JSON::ParserError] + def parser_error + ::JSON::ParserError + end + private # Convert JSON string into Ruby through toggleable adapters. @@ -134,14 +146,6 @@ module Gitlab opts end - # The standard parser error we should be returning. Defined in a method - # so we can potentially override it later. - # - # @return [JSON::ParserError] - def parser_error - ::JSON::ParserError - end - # @param [Nil, Boolean] an extracted :legacy_mode key from the opts hash # @return [Boolean] def legacy_mode_enabled?(arg_value) diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb index 41c18f82a4b..d5c018cfc68 100644 --- a/lib/gitlab/json_cache.rb +++ b/lib/gitlab/json_cache.rb @@ -2,12 +2,17 @@ module Gitlab class JsonCache - attr_reader :backend, :cache_key_with_version, :namespace + attr_reader :backend, :namespace + + STRATEGY_KEY_COMPONENTS = { + revision: Gitlab.revision, + version: [Gitlab::VERSION, Rails.version] + }.freeze def initialize(options = {}) @backend = options.fetch(:backend, Rails.cache) @namespace = options.fetch(:namespace, nil) - @cache_key_with_version = options.fetch(:cache_key_with_version, true) + @cache_key_strategy = options.fetch(:cache_key_strategy, :revision) end def active? @@ -19,13 +24,12 @@ module Gitlab end def cache_key(key) - expanded_cache_key = [namespace, key].compact - - if cache_key_with_version - expanded_cache_key << [Gitlab::VERSION, Rails.version] - end + expanded_cache_key = [namespace, key, *strategy_key_component].compact + expanded_cache_key.join(':').freeze + end - expanded_cache_key.flatten.join(':').freeze + def strategy_key_component + STRATEGY_KEY_COMPONENTS.fetch(@cache_key_strategy) end def expire(key) @@ -39,7 +43,9 @@ module Gitlab end def write(key, value, options = nil) - backend.write(cache_key(key), value.to_json, options) + # As we use json as the serialization format, return everything from + # ActiveModel objects included encrypted values. + backend.write(cache_key(key), value.to_json(unsafe_serialization_hash: true), options) end def fetch(key, options = {}, &block) diff --git a/lib/gitlab/kubernetes/kubeconfig/template.rb b/lib/gitlab/kubernetes/kubeconfig/template.rb index da0861ee86a..d40b9ce117e 100644 --- a/lib/gitlab/kubernetes/kubeconfig/template.rb +++ b/lib/gitlab/kubernetes/kubeconfig/template.rb @@ -14,6 +14,7 @@ module Gitlab @clusters = [] @users = [] @contexts = [] + @current_context = nil end def valid? @@ -32,14 +33,45 @@ module Gitlab contexts << new_entry(:context, **args) end + def merge_yaml(kubeconfig_yaml) + return unless kubeconfig_yaml + + kubeconfig_yaml = YAML.safe_load(kubeconfig_yaml, symbolize_names: true) + kubeconfig_yaml[:users].each do |user| + add_user( + name: user[:name], + token: user.dig(:user, :token) + ) + end + kubeconfig_yaml[:clusters].each do |cluster| + ca_pem = cluster.dig(:cluster, :'certificate-authority-data')&.yield_self do |data| + Base64.strict_decode64(data) + end + + add_cluster( + name: cluster[:name], + url: cluster.dig(:cluster, :server), + ca_pem: ca_pem + ) + end + kubeconfig_yaml[:contexts].each do |context| + add_context( + name: context[:name], + **context[:context]&.slice(:cluster, :user, :namespace) + ) + end + @current_context = kubeconfig_yaml[:'current-context'] + end + def to_h { apiVersion: 'v1', kind: 'Config', clusters: clusters.map(&:to_h), users: users.map(&:to_h), - contexts: contexts.map(&:to_h) - } + contexts: contexts.map(&:to_h), + 'current-context': current_context + }.compact end def to_yaml @@ -48,7 +80,7 @@ module Gitlab private - attr_reader :clusters, :users, :contexts + attr_reader :clusters, :users, :contexts, :current_context def new_entry(entry, **args) ENTRIES.fetch(entry).new(**args) diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb index 6f7fa9fe03b..b259f58350b 100644 --- a/lib/gitlab/language_detection.rb +++ b/lib/gitlab/language_detection.rb @@ -63,7 +63,7 @@ module Gitlab @repository .languages .first(MAX_LANGUAGES) - .to_h { |l| [l[:label], l] } + .index_by { |l| l[:label] } end end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index e93a297cee4..ef5ca56a13b 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -12,6 +12,11 @@ module Gitlab module MailRoom RAILS_ROOT_DIR = Pathname.new('../..').expand_path(__dir__).freeze + DELIVERY_METHOD_SIDEKIQ = 'sidekiq' + DELIVERY_METHOD_WEBHOOK = 'webhook' + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Mailroom-Api-Request' + INTERNAL_API_REQUEST_JWT_ISSUER = 'gitlab-mailroom' + DEFAULT_CONFIG = { enabled: false, port: 143, @@ -20,7 +25,8 @@ module Gitlab mailbox: 'inbox', idle_timeout: 60, log_path: RAILS_ROOT_DIR.join('log', 'mail_room_json.log'), - expunge_deleted: false + expunge_deleted: false, + delivery_method: DELIVERY_METHOD_SIDEKIQ }.freeze # Email specific configuration which is merged with configuration @@ -63,7 +69,9 @@ module Gitlab return {} unless File.exist?(config_file) config = merged_configs(config_key) + config.merge!(redis_config) if enabled?(config) + config[:log_path] = File.expand_path(config[:log_path], RAILS_ROOT_DIR) config diff --git a/lib/gitlab/mail_room/authenticator.rb b/lib/gitlab/mail_room/authenticator.rb index 26ebdca8beb..ca583d4cddb 100644 --- a/lib/gitlab/mail_room/authenticator.rb +++ b/lib/gitlab/mail_room/authenticator.rb @@ -6,8 +6,6 @@ module Gitlab include JwtAuthenticatable SecretConfigurationError = Class.new(StandardError) - INTERNAL_API_REQUEST_HEADER = 'Gitlab-Mailroom-Api-Request' - INTERNAL_API_REQUEST_JWT_ISSUER = 'gitlab-mailroom' # Only allow token generated within the last 5 minutes EXPIRATION = 5.minutes @@ -18,9 +16,10 @@ module Gitlab return false if enabled_configs[mailbox_type].blank? decode_jwt( - request_headers[INTERNAL_API_REQUEST_HEADER], + request_headers[Gitlab::MailRoom::INTERNAL_API_REQUEST_HEADER], secret(mailbox_type), - issuer: INTERNAL_API_REQUEST_JWT_ISSUER, iat_after: Time.current - EXPIRATION + issuer: Gitlab::MailRoom::INTERNAL_API_REQUEST_JWT_ISSUER, + iat_after: Time.current - EXPIRATION ) rescue JWT::DecodeError => e ::Gitlab::AppLogger.warn("Fail to decode MailRoom JWT token: #{e.message}") if Rails.env.development? diff --git a/lib/gitlab/merge_requests/commit_message_generator.rb b/lib/gitlab/merge_requests/commit_message_generator.rb index 0515c17fe5d..ef5c63925c2 100644 --- a/lib/gitlab/merge_requests/commit_message_generator.rb +++ b/lib/gitlab/merge_requests/commit_message_generator.rb @@ -50,6 +50,19 @@ module Gitlab .except(commit_author&.commit_email_or_default) .map { |author_email, author_name| "Co-authored-by: #{author_name} <#{author_email}>" } .join("\n") + end, + 'all_commits' => -> (merge_request, _, _) do + merge_request + .recent_commits + .without_merge_commits + .map do |commit| + if commit.safe_message&.bytesize&.>(100.kilobytes) + "* #{commit.title}\n\n-- Skipped commit body exceeding 100KiB in size." + else + "* #{commit.safe_message&.strip}" + end + end + .join("\n\n") end }.freeze diff --git a/lib/gitlab/merge_requests/mergeability/check_result.rb b/lib/gitlab/merge_requests/mergeability/check_result.rb index d0788c7d7c7..5284d20d423 100644 --- a/lib/gitlab/merge_requests/mergeability/check_result.rb +++ b/lib/gitlab/merge_requests/mergeability/check_result.rb @@ -22,8 +22,8 @@ module Gitlab def self.from_hash(data) new( - status: data.fetch(:status), - payload: data.fetch(:payload)) + status: data.fetch('status').to_sym, + payload: data.fetch('payload')) end def initialize(status:, payload: {}) diff --git a/lib/gitlab/merge_requests/mergeability/results_store.rb b/lib/gitlab/merge_requests/mergeability/results_store.rb index bb6489f8526..2f7b8888b2f 100644 --- a/lib/gitlab/merge_requests/mergeability/results_store.rb +++ b/lib/gitlab/merge_requests/mergeability/results_store.rb @@ -9,7 +9,11 @@ module Gitlab end def read(merge_check:) - interface.retrieve_check(merge_check: merge_check) + result_hash = interface.retrieve_check(merge_check: merge_check) + + return if result_hash.blank? + + CheckResult.from_hash(result_hash) end def write(merge_check:, result_hash:) diff --git a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb index 2c17982d299..31d75225972 100644 --- a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb @@ -74,7 +74,7 @@ module Gitlab def verify_params raise Errors::DashboardProcessingError, _('Cluster is required for Stages::ClusterEndpointInserter') unless params[:cluster] - raise Errors::DashboardProcessingError, _('Cluster type must be specificed for Stages::ClusterEndpointInserter') unless params[:cluster_type] + raise Errors::DashboardProcessingError, _('Cluster type must be specified for Stages::ClusterEndpointInserter') unless params[:cluster_type] end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 715dd86d93c..12576cabb19 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -134,11 +134,7 @@ module Gitlab :"gitlab_transaction_db_#{counter}_total" end - if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] - current_transaction&.increment(prometheus_key, 1, { db_config_name: db_config_name }) - else - current_transaction&.increment(prometheus_key, 1) - end + current_transaction&.increment(prometheus_key, 1, { db_config_name: db_config_name }) Gitlab::SafeRequestStore[log_key] = Gitlab::SafeRequestStore[log_key].to_i + 1 @@ -154,11 +150,7 @@ module Gitlab def observe(histogram, event, &block) db_config_name = db_config_name(event.payload) - if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] - current_transaction&.observe(histogram, event.duration / 1000.0, { db_config_name: db_config_name }, &block) - else - current_transaction&.observe(histogram, event.duration / 1000.0, &block) - end + current_transaction&.observe(histogram, event.duration / 1000.0, { db_config_name: db_config_name }, &block) end def current_transaction @@ -193,11 +185,9 @@ module Gitlab counters << compose_metric_key(metric, role) end - if ENV['GITLAB_MULTIPLE_DATABASE_METRICS'] - ::Gitlab::Database.db_config_names.each do |config_name| - counters << compose_metric_key(metric, nil, config_name) # main - counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica - end + ::Gitlab::Database.db_config_names.each do |config_name| + counters << compose_metric_key(metric, nil, config_name) # main + counters << compose_metric_key(metric, nil, config_name + ::Gitlab::Database::LoadBalancing::LoadBalancer::REPLICA_SUFFIX) # main_replica end end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index a9ff186c7cb..f4984e11c14 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -3,6 +3,7 @@ module Gitlab class OmniauthInitializer OAUTH2_TIMEOUT_SECONDS = 10 + ConfigurationError = Class.new(StandardError) def initialize(devise_config) @devise_config = devise_config @@ -75,16 +76,29 @@ module Gitlab provider_arguments << provider[argument] if provider[argument] end - case provider['args'] + arguments = provider.fetch('args', {}) + defaults = provider_defaults(provider) + + case arguments when Array - # An Array from the configuration will be expanded. - provider_arguments.concat provider['args'] + # An Array from the configuration will be expanded + provider_arguments.concat arguments + provider_arguments << defaults unless defaults.empty? when Hash - defaults = provider_defaults(provider) - hash_arguments = provider['args'].deep_symbolize_keys.deep_merge(defaults) + hash_arguments = arguments.deep_symbolize_keys.deep_merge(defaults) + normalized = normalize_hash_arguments(hash_arguments) # A Hash from the configuration will be passed as is. - provider_arguments << normalize_hash_arguments(hash_arguments) + provider_arguments << normalized unless normalized.empty? + else + # this will prevent the application from starting in development mode. + # we still set defaults, and let the application start in prod. + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + ConfigurationError.new("Arguments were provided for #{provider['name']}, but not as an array or a hash"), + provider_name: provider['name'], + arguments_type: arguments.class.name + ) + provider_arguments << defaults unless defaults.empty? end provider_arguments diff --git a/lib/gitlab/pages/settings.rb b/lib/gitlab/pages/settings.rb index b35683c9dec..7ada15cfe9a 100644 --- a/lib/gitlab/pages/settings.rb +++ b/lib/gitlab/pages/settings.rb @@ -16,7 +16,7 @@ module Gitlab def disk_access_denied? return true unless ::Settings.pages.local_store&.enabled - ::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite? + ::Gitlab::Runtime.puma? && !::Gitlab::Runtime.test_suite? end def report_denied_disk_access diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index 99a3145104a..e76cab688cc 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -4,6 +4,7 @@ module Gitlab module Pagination class GitalyKeysetPager attr_reader :request_context, :project + delegate :params, to: :request_context def initialize(request_context, project) diff --git a/lib/gitlab/pagination/keyset/cursor_based_request_context.rb b/lib/gitlab/pagination/keyset/cursor_based_request_context.rb index 18390f5b59d..e06d7e48ca3 100644 --- a/lib/gitlab/pagination/keyset/cursor_based_request_context.rb +++ b/lib/gitlab/pagination/keyset/cursor_based_request_context.rb @@ -6,6 +6,7 @@ module Gitlab class CursorBasedRequestContext DEFAULT_SORT_DIRECTION = :desc attr_reader :request_context + delegate :params, to: :request_context def initialize(request_context) diff --git a/lib/gitlab/pagination/keyset/header_builder.rb b/lib/gitlab/pagination/keyset/header_builder.rb index 888d93d5fe3..1036916e665 100644 --- a/lib/gitlab/pagination/keyset/header_builder.rb +++ b/lib/gitlab/pagination/keyset/header_builder.rb @@ -5,6 +5,7 @@ module Gitlab module Keyset class HeaderBuilder attr_reader :request_context + delegate :params, :header, :request, to: :request_context def initialize(request_context) diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 4f8a6ffb2cc..fca75d1fe01 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -4,6 +4,7 @@ module Gitlab module Pagination class OffsetPagination < Base attr_reader :request_context + delegate :params, :header, :request, to: :request_context def initialize(request_context) @@ -26,7 +27,7 @@ module Gitlab end return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) - return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops) + return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit, type: :ops, default_enabled: :yaml) limited_total_count = pagination_data.total_count_with_limit if limited_total_count > Kaminari::ActiveRecordRelationMethods::MAX_COUNT_LIMIT diff --git a/lib/gitlab/patch/action_cable_redis_listener.rb b/lib/gitlab/patch/action_cable_redis_listener.rb new file mode 100644 index 00000000000..b21bee45991 --- /dev/null +++ b/lib/gitlab/patch/action_cable_redis_listener.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# Modifies https://github.com/rails/rails/blob/v6.1.4.6/actioncable/lib/action_cable/subscription_adapter/redis.rb +# so that it is resilient to Redis connection errors. +# See https://github.com/rails/rails/issues/27659. + +# rubocop:disable Gitlab/ModuleWithInstanceVariables +module Gitlab + module Patch + module ActionCableRedisListener + private + + def ensure_listener_running + @thread ||= Thread.new do + Thread.current.abort_on_exception = true + + conn = @adapter.redis_connection_for_subscriptions + listen conn + rescue ::Redis::BaseConnectionError + @thread = @raw_client = nil + ::ActionCable.server.restart + end + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index 06a26c4830f..6f497c6376d 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -255,7 +255,7 @@ module Gitlab end def container_image_regex - @container_image_regex ||= %r{([\w\.-]+\/){0,1}[\w\.-]+}.freeze + @container_image_regex ||= %r{([\w\.-]+\/){0,4}[\w\.-]+}.freeze end def container_image_blob_sha_regex diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb index cf524e69454..8743772eef6 100644 --- a/lib/gitlab/performance_bar/stats.rb +++ b/lib/gitlab/performance_bar/stats.rb @@ -25,8 +25,8 @@ module Gitlab log_queries(id, data, 'active-record') log_queries(id, data, 'gitaly') log_queries(id, data, 'redis') - rescue StandardError => err - logger.error(message: "failed to process request id #{id}: #{err.message}") + rescue StandardError => e + logger.error(message: "failed to process request id #{id}: #{e.message}") end private @@ -34,6 +34,8 @@ module Gitlab def request(id) # Peek gem stores request data under peek:requests:request_id key json_data = @redis.get("peek:requests:#{id}") + raise "No data for #{id}" if json_data.nil? + Gitlab::Json.parse(json_data) end diff --git a/lib/gitlab/process_supervisor.rb b/lib/gitlab/process_supervisor.rb new file mode 100644 index 00000000000..18fd24aa582 --- /dev/null +++ b/lib/gitlab/process_supervisor.rb @@ -0,0 +1,149 @@ +# frozen_string_literal: true + +module Gitlab + # Given a set of process IDs, the supervisor can monitor processes + # for being alive and invoke a callback if some or all should go away. + # The receiver of the callback can then act on this event, for instance + # by restarting those processes or performing clean-up work. + # + # The supervisor will also trap termination signals if provided and + # propagate those to the supervised processes. Any supervised processes + # that do not terminate within a specified grace period will be killed. + class ProcessSupervisor < Gitlab::Daemon + DEFAULT_HEALTH_CHECK_INTERVAL_SECONDS = 5 + DEFAULT_TERMINATE_INTERVAL_SECONDS = 1 + DEFAULT_TERMINATE_TIMEOUT_SECONDS = 10 + + attr_reader :alive + + def initialize( + health_check_interval_seconds: DEFAULT_HEALTH_CHECK_INTERVAL_SECONDS, + check_terminate_interval_seconds: DEFAULT_TERMINATE_INTERVAL_SECONDS, + terminate_timeout_seconds: DEFAULT_TERMINATE_TIMEOUT_SECONDS, + term_signals: %i(INT TERM), + forwarded_signals: [], + **options) + super(**options) + + @term_signals = term_signals + @forwarded_signals = forwarded_signals + @health_check_interval_seconds = health_check_interval_seconds + @check_terminate_interval_seconds = check_terminate_interval_seconds + @terminate_timeout_seconds = terminate_timeout_seconds + + @pids = [] + @alive = false + end + + # Starts a supervision loop for the given process ID(s). + # + # If any or all processes go away, the IDs of any dead processes will + # be yielded to the given block, so callers can act on them. + # + # If the block returns a non-empty list of IDs, the supervisor will + # start observing those processes instead. Otherwise it will shut down. + def supervise(pid_or_pids, &on_process_death) + @pids = Array(pid_or_pids) + @on_process_death = on_process_death + + trap_signals! + + start + end + + # Shuts down the supervisor and all supervised processes with the given signal. + def shutdown(signal = :TERM) + return unless @alive + + stop_processes(signal) + stop + end + + def supervised_pids + @pids + end + + private + + def start_working + @alive = true + end + + def stop_working + @alive = false + end + + def run_thread + while @alive + sleep(@health_check_interval_seconds) + + check_process_health + end + end + + def check_process_health + unless all_alive? + existing_pids = live_pids # Capture this value for the duration of the block. + dead_pids = @pids - existing_pids + new_pids = Array(@on_process_death.call(dead_pids)) + @pids = existing_pids + new_pids + @alive = @pids.any? + end + end + + def stop_processes(signal) + # Set this prior to shutting down so that shutdown hooks which read `alive` + # know the supervisor is about to shut down. + @alive = false + + # Shut down supervised processes. + signal_all(signal) + wait_for_termination + end + + def trap_signals! + ProcessManagement.trap_signals(@term_signals) do |signal| + stop_processes(signal) + end + + ProcessManagement.trap_signals(@forwarded_signals) do |signal| + signal_all(signal) + end + end + + def wait_for_termination + deadline = monotonic_time + @terminate_timeout_seconds + sleep(@check_terminate_interval_seconds) while continue_waiting?(deadline) + + hard_stop_stuck_pids + end + + def monotonic_time + Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) + end + + def continue_waiting?(deadline) + any_alive? && monotonic_time < deadline + end + + def signal_all(signal) + ProcessManagement.signal_processes(@pids, signal) + end + + def hard_stop_stuck_pids + ProcessManagement.signal_processes(live_pids, "-KILL") + end + + def any_alive? + ProcessManagement.any_alive?(@pids) + end + + def all_alive? + ProcessManagement.all_alive?(@pids) + end + + def live_pids + ProcessManagement.pids_alive(@pids) + end + end +end diff --git a/lib/gitlab/profiler.rb b/lib/gitlab/profiler.rb index b2179d80a18..3a5f1a1d480 100644 --- a/lib/gitlab/profiler.rb +++ b/lib/gitlab/profiler.rb @@ -28,7 +28,7 @@ module Gitlab ].freeze # Takes a URL to profile (can be a fully-qualified URL, or an absolute path) - # and returns the ruby-prof profile result. Formatting that result is the + # and returns the profiler result. Formatting that result is the # caller's responsibility. Requests are GET requests unless post_data is # passed. # @@ -43,7 +43,13 @@ module Gitlab # # - private_token: instead of providing a user instance, the token can be # given as a string. Takes precedence over the user option. - def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil) + # + # - sampling_mode: When true, uses a sampling profiler (StackProf) instead of a tracing profiler (RubyProf). + # + # - profiler_options: A keyword Hash of arguments passed to the profiler. Defaults by profiler type: + # RubyProf - {} + # StackProf - { mode: :wall, out: <some temporary file>, interval: 1000, raw: true } + def self.profile(url, logger: nil, post_data: nil, user: nil, private_token: nil, sampling_mode: false, profiler_options: {}) app = ActionDispatch::Integration::Session.new(Rails.application) verb = :get headers = {} @@ -75,7 +81,9 @@ module Gitlab with_custom_logger(logger) do with_user(user) do - RubyProf.profile { app.public_send(verb, url, params: post_data, headers: headers) } # rubocop:disable GitlabSecurity/PublicSend + with_profiler(sampling_mode, profiler_options) do + app.public_send(verb, url, params: post_data, headers: headers) # rubocop:disable GitlabSecurity/PublicSend + end end end end @@ -172,5 +180,16 @@ module Gitlab RubyProf::FlatPrinter.new(result).print($stdout, default_options.merge(options)) end + + def self.with_profiler(sampling_mode, profiler_options) + if sampling_mode + require 'stackprof' + args = { mode: :wall, interval: 1000, raw: true }.merge!(profiler_options) + args[:out] ||= ::Tempfile.new(["profile-#{Time.now.to_i}-", ".dump"]).path + ::StackProf.run(**args) { yield } + else + RubyProf.profile(**profiler_options) { yield } + end + end end end diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb index 121626ced56..1d7b179baf0 100644 --- a/lib/gitlab/project_authorizations.rb +++ b/lib/gitlab/project_authorizations.rb @@ -22,7 +22,7 @@ module Gitlab user.projects_with_active_memberships.select_for_project_authorization, # The personal projects of the user. - user.personal_projects.select_as_maintainer_for_project_authorization, + user.personal_projects.select_project_owner_for_project_authorization, # Projects that belong directly to any of the groups the user has # access to. diff --git a/lib/gitlab/prometheus/queries/base_query.rb b/lib/gitlab/prometheus/queries/base_query.rb index 9ff414d5236..eabac6128b5 100644 --- a/lib/gitlab/prometheus/queries/base_query.rb +++ b/lib/gitlab/prometheus/queries/base_query.rb @@ -5,6 +5,7 @@ module Gitlab module Queries class BaseQuery attr_accessor :client + delegate :query_range, :query, :label_values, :series, to: :client, prefix: true def raw_memory_usage_query(environment_slug) diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index b44b47eca37..2f89774a257 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -291,7 +291,7 @@ module Gitlab types Issue condition do current_user.can?(:set_issue_crm_contacts, quick_action_target) && - CustomerRelations::Contact.exists_for_group?(quick_action_target.project.group) + CustomerRelations::Contact.exists_for_group?(quick_action_target.project.root_ancestor) end execution_message do _('One or more contacts were successfully added.') @@ -306,7 +306,7 @@ module Gitlab types Issue condition do current_user.can?(:set_issue_crm_contacts, quick_action_target) && - CustomerRelations::Contact.exists_for_group?(quick_action_target.project.group) + CustomerRelations::Contact.exists_for_group?(quick_action_target.project.root_ancestor) end execution_message do _('One or more contacts were successfully removed.') diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 842d4ef482b..e6a73c71e85 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -23,7 +23,11 @@ module Gitlab end end execution_message do - if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) + if params[:merge_request_diff_head_sha].blank? + _("Merge request diff sha parameter is required for the merge quick action.") + elsif params[:merge_request_diff_head_sha] != quick_action_target.diff_head_sha + _("Branch has been updated since the merge was requested.") + elsif preferred_strategy = preferred_auto_merge_strategy(quick_action_target) _("Scheduled to merge this merge request (%{strategy}).") % { strategy: preferred_strategy.humanize } else _('Merged this merge request.') @@ -35,6 +39,10 @@ module Gitlab merge_orchestration_service.can_merge?(quick_action_target) end command :merge do + next unless params[:merge_request_diff_head_sha].present? + + next unless params[:merge_request_diff_head_sha] == quick_action_target.diff_head_sha + @updates[:merge] = params[:merge_request_diff_head_sha] end diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index a6491d23bf5..c9202c6c54c 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -259,6 +259,15 @@ module Gitlab "It must start with a letter, digit, emoji, or '_'." end + # Project path must conform to this regex. See https://gitlab.com/gitlab-org/gitlab/-/issues/27483 + def oci_repository_path_regex + @oci_repository_path_regex ||= %r{\A[a-zA-Z0-9]+([._-][a-zA-Z0-9]+)*\z}.freeze + end + + def oci_repository_path_regex_message + 'must not start or end with a special character and must not contain consecutive special characters.' + end + def group_name_regex @group_name_regex ||= /\A#{group_name_regex_chars}\z/.freeze end @@ -459,6 +468,15 @@ module Gitlab "can contain only lowercase letters, digits, '_' and '-'. " \ "Must start with a letter, and cannot end with '-' or '_'" end + + def saved_reply_name_regex + @saved_reply_name_regex ||= /\A[a-z]([a-z0-9\-_]*[a-z0-9])?\z/.freeze + end + + def saved_reply_name_regex_message + "can contain only lowercase letters, digits, '_' and '-'. " \ + "Must start with a letter, and cannot end with '-' or '_'" + end end end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 574e05658bc..5b1341207fd 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -65,12 +65,15 @@ module Gitlab !!defined?(::Rails::Command::RunnerCommand) end - def web_server? - puma? + # Whether we are executing in an actual application context i.e. Puma or Sidekiq. + def application? + puma? || sidekiq? end + # Whether we are executing in a multi-threaded environment. For now this is equivalent + # to meaning Puma or Sidekiq, but this could change in the future. def multi_threaded? - puma? || sidekiq? + application? end def puma_in_clustered_mode? @@ -94,7 +97,7 @@ module Gitlab threads += Sidekiq.options[:concurrency] + 2 end - if web_server? + if puma? threads += Gitlab::ActionCable::Config.worker_pool_size end diff --git a/lib/gitlab/safe_request_loader.rb b/lib/gitlab/safe_request_loader.rb new file mode 100644 index 00000000000..89eca16c272 --- /dev/null +++ b/lib/gitlab/safe_request_loader.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + class SafeRequestLoader + def self.execute(args, &block) + new(**args).execute(&block) + end + + def initialize(resource_key:, resource_ids:, default_value: nil) + @resource_key = resource_key + @resource_ids = resource_ids.uniq + @default_value = default_value + @resource_data = {} + end + + def execute(&block) + raise ArgumentError, 'Block is mandatory' unless block_given? + + load_resource_data + remove_loaded_resource_ids + + update_resource_data(&block) + + resource_data + end + + private + + attr_reader :resource_key, :resource_ids, :default_value, :resource_data, :missing_resource_ids + + def load_resource_data + @resource_data = Gitlab::SafeRequestStore.fetch(resource_key) { resource_data } + end + + def remove_loaded_resource_ids + # Look up only the IDs we need + @missing_resource_ids = resource_ids - resource_data.keys + end + + def update_resource_data(&block) + return if missing_resource_ids.blank? + + reloaded_resource_data = yield(missing_resource_ids) + + @resource_data.merge!(reloaded_resource_data) + + mark_absent_values + end + + def mark_absent_values + absent = (missing_resource_ids - resource_data.keys).to_h { [_1, default_value] } + @resource_data.merge!(absent) + end + end +end diff --git a/lib/gitlab/sanitizers/exif.rb b/lib/gitlab/sanitizers/exif.rb index f607aff9d29..e302729df66 100644 --- a/lib/gitlab/sanitizers/exif.rb +++ b/lib/gitlab/sanitizers/exif.rb @@ -97,6 +97,28 @@ module Gitlab end end + def clean_existing_path(src_path, dry_run: false, content: nil, skip_unallowed_types: false) + content ||= File.read(src_path) + + if skip_unallowed_types + return unless check_for_allowed_types(content, raise_error: false) + else + check_for_allowed_types(content) + end + + to_remove = extra_tags(src_path) + + if to_remove.empty? + logger.info "#{src_path}: only whitelisted tags present, skipping" + return + end + + logger.info "#{src_path}: found exif tags to remove: #{to_remove}" + return if dry_run + + exec_remove_exif!(src_path) + end + private def extra_tags(path) @@ -146,12 +168,15 @@ module Gitlab filename end - def check_for_allowed_types(contents) + def check_for_allowed_types(contents, raise_error: true) mime_type = Gitlab::Utils::MimeType.from_string(contents) - unless ALLOWED_MIME_TYPES.include?(mime_type) + allowed = ALLOWED_MIME_TYPES.include?(mime_type) + if !allowed && raise_error raise "File type #{mime_type} not supported. Only supports #{ALLOWED_MIME_TYPES.join(", ")}." end + + allowed end def upload_ref(upload) diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 5671fce481f..e2df60c46f1 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -62,10 +62,6 @@ module Gitlab end def self.quiet - # Disable database insertion logs so speed isn't limited by ability to print to console - old_logger = ActiveRecord::Base.logger - ActiveRecord::Base.logger = nil - # Additional seed logic for models. Project.include(ProjectSeed) User.include(UserSeed) @@ -75,9 +71,11 @@ module Gitlab SeedFu.quiet = true - without_statement_timeout do - without_new_note_notifications do - yield + without_database_logging do + without_statement_timeout do + without_new_note_notifications do + yield + end end end @@ -85,7 +83,6 @@ module Gitlab ensure SeedFu.quiet = false ActionMailer::Base.perform_deliveries = old_perform_deliveries - ActiveRecord::Base.logger = old_logger end def self.without_gitaly_timeout @@ -112,10 +109,30 @@ module Gitlab end def self.without_statement_timeout - ActiveRecord::Base.connection.execute('SET statement_timeout=0') + Gitlab::Database::EachDatabase.each_database_connection do |connection| + connection.execute('SET statement_timeout=0') + end + yield + ensure + Gitlab::Database::EachDatabase.each_database_connection do |connection| + connection.execute('RESET statement_timeout') + end + end + + def self.without_database_logging + old_loggers = Gitlab::Database.database_base_models.transform_values do |model| + model.logger + end + + Gitlab::Database.database_base_models.each do |_, model| + model.logger = nil + end + yield ensure - ActiveRecord::Base.connection.execute('RESET statement_timeout') + Gitlab::Database.database_base_models.each do |connection_name, model| + model.logger = old_loggers[connection_name] + end end end end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index 69802fd6217..fd3a5f715e8 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -33,7 +33,7 @@ module Gitlab chain.add ::Gitlab::SidekiqMiddleware::BatchLoader chain.add ::Gitlab::SidekiqMiddleware::InstrumentationLogger chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Server - chain.add ::Gitlab::SidekiqMiddleware::QueryAnalyzer if Gitlab.dev_or_test_env? || Gitlab::Utils.to_boolean(ENV['GITLAB_ENABLE_QUERY_ANALYZERS'], default: false) + chain.add ::Gitlab::SidekiqMiddleware::QueryAnalyzer chain.add ::Gitlab::SidekiqVersioning::Middleware chain.add ::Gitlab::SidekiqStatus::ServerMiddleware chain.add ::Gitlab::SidekiqMiddleware::WorkerContext::Server diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index f31262bfcc9..601c8d1c3cf 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -167,7 +167,6 @@ module Gitlab def idempotent? return false unless worker_klass return false unless worker_klass.respond_to?(:idempotent?) - return false unless preserve_wal_location? || !worker_klass.utilizes_load_balancing_capabilities? worker_klass.idempotent? end @@ -206,8 +205,6 @@ module Gitlab end def job_wal_locations - return {} unless preserve_wal_location? - job['wal_locations'] || {} end @@ -272,10 +269,6 @@ module Gitlab @existing_wal_locations ||= {} end - def preserve_wal_location? - Feature.enabled?(:preserve_latest_wal_locations_for_idempotent_jobs, default_enabled: :yaml) - end - def reschedulable? !scheduled? && options[:if_deduplicated] == :reschedule_once end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index bea98403997..f3e1d0af2aa 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -7,18 +7,26 @@ module Gitlab # SIDEKIQ_LATENCY_BUCKETS are latency histogram buckets better suited to Sidekiq # timeframes than the DEFAULT_BUCKET definition. Defined in seconds. - SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60, 300, 600].freeze + # This information is better viewed in logs, but these buckets cover + # most of the durations for cpu, gitaly, db and elasticsearch + SIDEKIQ_LATENCY_BUCKETS = [0.1, 0.5, 1, 2.5].freeze + + # These are the buckets we currently use for alerting, we will likely + # replace these histograms with Application SLIs + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1313 + SIDEKIQ_JOB_DURATION_BUCKETS = [10, 300].freeze + SIDEKIQ_QUEUE_DURATION_BUCKETS = [10, 60].freeze class << self include ::Gitlab::SidekiqMiddleware::MetricsHelper def metrics { - sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_JOB_DURATION_BUCKETS), sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_QUEUE_DURATION_BUCKETS), sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS), sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb index 3de6c8df8aa..acc3e1712ab 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -29,7 +29,12 @@ module Gitlab # The worker classes aren't constants here, because that would force # Application Settings to be loaded earlier causing failures loading # the environment in rake tasks - EXEMPT_WORKER_NAMES = %w[BackgroundMigrationWorker BackgroundMigration::CiDatabaseWorker Database::BatchedBackgroundMigrationWorker].to_set + + EXEMPT_WORKER_NAMES = %w[BackgroundMigrationWorker + BackgroundMigration::CiDatabaseWorker + Database::BatchedBackgroundMigrationWorker + Database::BatchedBackgroundMigration::CiDatabaseWorker].to_set + JOB_STATUS_KEY = 'size_limiter' class << self diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 09236a7f1f0..c0730e7bd59 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -61,6 +61,16 @@ module Gitlab def self.with_fallback(pattern, multiline: false) UntrustedRegexp.new(pattern, multiline: multiline) rescue RegexpError + raise if Feature.enabled?(:disable_unsafe_regexp, default_enabled: :yaml) + + if Feature.enabled?(:ci_unsafe_regexp_logger, type: :ops, default_enabled: :yaml) + Gitlab::AppJsonLogger.info( + class: self.name, + regexp: pattern.to_s, + fabricated: 'unsafe ruby regexp' + ) + end + Regexp.new(pattern) end diff --git a/lib/gitlab/untrusted_regexp/ruby_syntax.rb b/lib/gitlab/untrusted_regexp/ruby_syntax.rb index 5176a6f6273..1f1da592ce0 100644 --- a/lib/gitlab/untrusted_regexp/ruby_syntax.rb +++ b/lib/gitlab/untrusted_regexp/ruby_syntax.rb @@ -16,40 +16,23 @@ module Gitlab # The regexp can match the pattern `/.../`, but may not be fabricatable: # it can be invalid or incomplete: `/match ( string/` - def self.valid?(pattern, fallback: false) - !!self.fabricate(pattern, fallback: fallback) + def self.valid?(pattern) + !!self.fabricate(pattern) end - def self.fabricate(pattern, fallback: false, project: nil) - self.fabricate!(pattern, fallback: fallback, project: project) + def self.fabricate(pattern, project: nil) + self.fabricate!(pattern, project: project) rescue RegexpError nil end - def self.fabricate!(pattern, fallback: false, project: nil) + def self.fabricate!(pattern, project: nil) raise RegexpError, 'Pattern is not string!' unless pattern.is_a?(String) matches = pattern.match(PATTERN) raise RegexpError, 'Invalid regular expression!' if matches.nil? - begin - create_untrusted_regexp(matches[:regexp], matches[:flags]) - rescue RegexpError - raise unless fallback && - Feature.enabled?(:allow_unsafe_ruby_regexp, default_enabled: :yaml) - - if Feature.enabled?(:ci_unsafe_regexp_logger, type: :ops, default_enabled: :yaml) - Gitlab::AppJsonLogger.info( - class: self.name, - regexp: pattern.to_s, - fabricated: 'unsafe ruby regexp', - project_id: project&.id, - project_path: project&.full_path - ) - end - - create_ruby_regexp(matches[:regexp], matches[:flags]) - end + create_untrusted_regexp(matches[:regexp], matches[:flags]) end def self.create_untrusted_regexp(pattern, flags) @@ -58,15 +41,6 @@ module Gitlab UntrustedRegexp.new(pattern, multiline: false) end private_class_method :create_untrusted_regexp - - def self.create_ruby_regexp(pattern, flags) - options = 0 - options += Regexp::IGNORECASE if flags&.include?('i') - options += Regexp::MULTILINE if flags&.include?('m') - - Regexp.new(pattern, options) - end - private_class_method :create_ruby_regexp end end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 48228ede684..fe8c2227659 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -13,6 +13,7 @@ module Gitlab # ports - Raises error if the given URL port does is not between given ports. # allow_localhost - Raises error if URL resolves to a localhost IP address and argument is false. # allow_local_network - Raises error if URL resolves to a link-local address and argument is false. + # allow_object_storage - Avoid raising an error if URL resolves to an object storage endpoint and argument is true. # ascii_only - Raises error if URL has unicode characters and argument is true. # enforce_user - Raises error if URL user doesn't start with alphanumeric characters and argument is true. # enforce_sanitization - Raises error if URL includes any HTML/CSS/JS tags and argument is true. @@ -25,6 +26,7 @@ module Gitlab schemes: [], allow_localhost: false, allow_local_network: true, + allow_object_storage: false, ascii_only: false, enforce_user: false, enforce_sanitization: false, @@ -58,6 +60,8 @@ module Gitlab # Allow url from the GitLab instance itself but only for the configured hostname and ports return protected_uri_with_hostname if internal?(uri) + return protected_uri_with_hostname if allow_object_storage && object_storage_endpoint?(uri) + validate_local_request( address_info: address_info, allow_localhost: allow_localhost, @@ -149,6 +153,7 @@ module Gitlab validate_local_network(address_info) validate_link_local(address_info) validate_shared_address(address_info) + validate_limited_broadcast_address(address_info) end end @@ -253,6 +258,17 @@ module Gitlab raise BlockedUrlError, "Requests to the link local network are not allowed" end + # Raises a BlockedUrlError if any IP in `addrs_info` is the limited + # broadcast address. + # https://datatracker.ietf.org/doc/html/rfc919#section-7 + def validate_limited_broadcast_address(addrs_info) + blocked_ips = ["255.255.255.255"] + + return if (blocked_ips & addrs_info.map(&:ip_address)).empty? + + raise BlockedUrlError, "Requests to the limited broadcast address are not allowed" + end + def internal?(uri) internal_web?(uri) || internal_shell?(uri) end @@ -269,6 +285,30 @@ module Gitlab get_port(uri) == config.gitlab_shell.ssh_port end + def enabled_object_storage_endpoints + ObjectStoreSettings::SUPPORTED_TYPES.collect do |type| + section_setting = config.try(type) + + next unless section_setting + + object_store_setting = section_setting['object_store'] + + next unless object_store_setting && object_store_setting['enabled'] + + object_store_setting.dig('connection', 'endpoint') + end.compact.uniq + end + + def object_storage_endpoint?(uri) + enabled_object_storage_endpoints.any? do |endpoint| + endpoint_uri = URI(endpoint) + + uri.scheme == endpoint_uri.scheme && + uri.hostname == endpoint_uri.hostname && + get_port(uri) == get_port(endpoint_uri) + end + end + def domain_allowed?(uri) Gitlab::UrlBlockers::UrlAllowlist.domain_allowed?(uri.normalized_host, port: get_port(uri)) end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 6e5196ecdbd..1031f38792b 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -80,6 +80,10 @@ module Gitlab @all ||= definitions.map { |_key_path, definition| definition } end + def not_removed + all.select { |definition| definition.attributes[:status] != 'removed' }.index_by(&:key_path) + end + def with_instrumentation_class all.select { |definition| definition.attributes[:instrumentation_class].present? && definition.available? } end diff --git a/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb new file mode 100644 index 00000000000..6df6fef5d3a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/cert_based_clusters_ff_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CertBasedClustersFfMetric < GenericMetric + value do + Feature.enabled?(:certificate_based_clusters, default_enabled: :yaml, type: :ops) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index d7fc798ebe2..34a8bfd08b5 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -33,6 +33,12 @@ module Gitlab @metric_relation = block end + def metric_options(&block) + return @metric_options&.call.to_h unless block_given? + + @metric_options = block + end + def operation(symbol, column: nil, &block) @metric_operation = symbol @column = column @@ -54,6 +60,7 @@ module Gitlab self.class.column, start: start, finish: finish, + **self.class.metric_options, &self.class.metric_operation_block) end diff --git a/lib/gitlab/usage/service_ping/instrumented_payload.rb b/lib/gitlab/usage/service_ping/instrumented_payload.rb new file mode 100644 index 00000000000..e04e2e589b2 --- /dev/null +++ b/lib/gitlab/usage/service_ping/instrumented_payload.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +# Service Ping payload build using the instrumentation classes +# for given metrics key_paths and output method +module Gitlab + module Usage + module ServicePing + class InstrumentedPayload + attr_reader :metrics_key_paths + attr_reader :output_method + + def initialize(metrics_key_paths, output_method) + @metrics_key_paths = metrics_key_paths + @output_method = output_method + end + + def build + metrics_key_paths.map do |key_path| + compute_instrumental_value(key_path, output_method) + end.reduce({}, :deep_merge) + end + + private + + # Not all metrics defintions have instrumentation classes + # The value can be computed only for those that have it + def instrumented_metrics_defintions + Gitlab::Usage::MetricDefinition.with_instrumentation_class + end + + def compute_instrumental_value(key_path, output_method) + definition = instrumented_metrics_defintions.find { |df| df.key_path == key_path } + + return {} unless definition.present? + + Gitlab::Usage::Metric.new(definition).method(output_method).call + end + end + end + end +end diff --git a/lib/gitlab/usage/service_ping/payload_keys_processor.rb b/lib/gitlab/usage/service_ping/payload_keys_processor.rb new file mode 100644 index 00000000000..ea2043ffb83 --- /dev/null +++ b/lib/gitlab/usage/service_ping/payload_keys_processor.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# Process the UsageData payload to get the keys that have a metric defintion +# Get the missing keys from the payload +module Gitlab + module Usage + module ServicePing + class PayloadKeysProcessor + attr_reader :old_payload + + def initialize(old_payload) + @old_payload = old_payload + end + + def key_paths + @key_paths ||= payload_keys.to_a.flatten.compact + end + + def missing_instrumented_metrics_key_paths + @missing_key_paths ||= metrics_with_instrumentation.map(&:key) - key_paths + end + + private + + def payload_keys(payload = old_payload, parents = []) + return unless payload.is_a?(Hash) + + payload.map do |key, value| + if has_metric_definition?(key, parents) + parents.dup.append(key).join('.') + else + payload_keys(value, parents.dup << key) if value.is_a?(Hash) + end + end + end + + def has_metric_definition?(key, parent_keys) + key_path = parent_keys.dup.append(key).join('.') + metric_definitions.key?(key_path) + end + + def metric_definitions + ::Gitlab::Usage::MetricDefinition.not_removed + end + + def metrics_with_instrumentation + ::Gitlab::Usage::MetricDefinition.with_instrumentation_class + end + end + end + end +end + +Gitlab::Usage::ServicePing::PayloadKeysProcessor.prepend_mod_with('Gitlab::Usage::ServicePing::PayloadKeysProcessor') diff --git a/lib/gitlab/usage/service_ping_report.rb b/lib/gitlab/usage/service_ping_report.rb index d9e30c46498..794f3373043 100644 --- a/lib/gitlab/usage/service_ping_report.rb +++ b/lib/gitlab/usage/service_ping_report.rb @@ -7,16 +7,29 @@ module Gitlab def for(output:, cached: false) case output.to_sym when :all_metrics_values - all_metrics_values(cached) + with_instrumentation_classes(all_metrics_values(cached), :with_value) when :metrics_queries - metrics_queries + with_instrumentation_classes(metrics_queries, :with_instrumentation) when :non_sql_metrics_values - non_sql_metrics_values + with_instrumentation_classes(non_sql_metrics_values, :with_instrumentation) end end private + def with_instrumentation_classes(old_payload, output_method) + if Feature.enabled?(:merge_service_ping_instrumented_metrics, default_enabled: :yaml) + + instrumented_metrics_key_paths = Gitlab::Usage::ServicePing::PayloadKeysProcessor.new(old_payload).missing_instrumented_metrics_key_paths + + instrumented_payload = Gitlab::Usage::ServicePing::InstrumentedPayload.new(instrumented_metrics_key_paths, output_method).build + + old_payload.deep_merge(instrumented_payload) + else + old_payload + end + end + def all_metrics_values(cached) Rails.cache.fetch('usage_data', force: !cached, expires_in: 2.weeks) do Gitlab::UsageData.data diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index e66a565246b..951ec5ea5c3 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -70,10 +70,9 @@ module Gitlab def system_usage_data issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)) - { + counts = { counts: { assignee_lists: count(List.assignee), - boards: add_metric('CountBoardsMetric', time_frame: 'all'), ci_builds: count(::Ci::Build), ci_internal_pipelines: count(::Ci::Pipeline.internal), ci_external_pipelines: count(::Ci::Pipeline.external), @@ -167,6 +166,12 @@ module Gitlab data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end } + + if Feature.disabled?(:merge_service_ping_instrumented_metrics, default_enabled: :yaml) + counts[:counts][:boards] = add_metric('CountBoardsMetric', time_frame: 'all') + end + + counts end # rubocop: enable Metrics/AbcSize @@ -219,7 +224,8 @@ module Gitlab collected_data_categories: add_metric('CollectedDataCategoriesMetric', time_frame: 'none'), service_ping_features_enabled: add_metric('ServicePingFeaturesMetric', time_frame: 'none'), snowplow_enabled: add_metric('SnowplowEnabledMetric', time_frame: 'none'), - snowplow_configured_to_gitlab_collector: add_metric('SnowplowConfiguredToGitlabCollectorMetric', time_frame: 'none') + snowplow_configured_to_gitlab_collector: add_metric('SnowplowConfiguredToGitlabCollectorMetric', time_frame: 'none'), + certificate_based_clusters_ff: add_metric('CertBasedClustersFfMetric') } } end diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb index cecc24a38d5..cdcad8fdc7b 100644 --- a/lib/gitlab/usage_data_counters.rb +++ b/lib/gitlab/usage_data_counters.rb @@ -16,7 +16,8 @@ module Gitlab DesignsCounter, KubernetesAgentCounter, StaticSiteEditorCounter, - DiffsCounter + DiffsCounter, + ServiceUsageDataCounter ].freeze UsageDataCounterError = Class.new(StandardError) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index c6e9db6a314..474ab9a4dd9 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -26,6 +26,7 @@ module Gitlab ecosystem epic_boards_usage epics_usage + error_tracking ide_edit incident_management issues_edit diff --git a/lib/gitlab/usage_data_counters/known_events/ci_users.yml b/lib/gitlab/usage_data_counters/known_events/ci_users.yml new file mode 100644 index 00000000000..63498a35858 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/ci_users.yml @@ -0,0 +1,5 @@ +- name: ci_users_executing_deployment_job + category: ci_users + redis_slot: ci_users + aggregation: weekly + feature_flag: job_deployment_count diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 96755db8439..fdf4bc58525 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -324,7 +324,6 @@ category: snippets redis_slot: snippets aggregation: weekly - feature_flag: usage_data_i_snippets_show # Terraform - name: p_terraform_state_api_unique_users category: terraform diff --git a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml new file mode 100644 index 00000000000..a56e0a6d370 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml @@ -0,0 +1,11 @@ +--- +- name: error_tracking_view_details + category: error_tracking + redis_slot: error_tracking + aggregation: weekly + feature_flag: track_error_tracking_activity +- name: error_tracking_view_list + category: error_tracking + redis_slot: error_tracking + aggregation: weekly + feature_flag: track_error_tracking_activity diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 49891080b03..4ba7ea2d407 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -127,6 +127,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_page + category: quickactions + redis_slot: quickactions + aggregation: weekly - name: i_quickactions_publish category: quickactions redis_slot: quickactions diff --git a/lib/gitlab/usage_data_counters/known_events/work_items.yml b/lib/gitlab/usage_data_counters/known_events/work_items.yml new file mode 100644 index 00000000000..0c9c6026c46 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/work_items.yml @@ -0,0 +1,11 @@ +--- +- name: users_updating_work_item_title + category: work_items + redis_slot: users + aggregation: weekly + feature_flag: track_work_items_activity +- name: users_creating_work_items + category: work_items + redis_slot: users + aggregation: weekly + feature_flag: track_work_items_activity diff --git a/lib/gitlab/usage_data_counters/service_usage_data_counter.rb b/lib/gitlab/usage_data_counters/service_usage_data_counter.rb new file mode 100644 index 00000000000..aa1d9583ea5 --- /dev/null +++ b/lib/gitlab/usage_data_counters/service_usage_data_counter.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class ServiceUsageDataCounter < BaseCounter + KNOWN_EVENTS = %w[download_payload_click].freeze + PREFIX = 'service_usage_data' + end +end diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb new file mode 100644 index 00000000000..6f5300405c7 --- /dev/null +++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module WorkItemActivityUniqueCounter + WORK_ITEM_CREATED = 'users_creating_work_items' + WORK_ITEM_TITLE_CHANGED = 'users_updating_work_item_title' + + class << self + def track_work_item_created_action(author:) + track_unique_action(WORK_ITEM_CREATED, author) + end + + def track_work_item_title_changed_action(author:) + track_unique_action(WORK_ITEM_TITLE_CHANGED, author) + end + + private + + def track_unique_action(action, author) + return unless author + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index c6490ba7374..d40ac71afc6 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -41,9 +41,19 @@ module Gitlab end def maximum_id(model, column = nil) + # no-op: shadowing super for performance reasons end def minimum_id(model, column = nil) + # no-op: shadowing super for performance reasons + end + + def alt_usage_data(value = nil, fallback: FALLBACK, &block) + if block_given? + { alt_usage_data_block: block.to_s } + else + { alt_usage_data_value: value } + end end def redis_usage_data(counter = nil, &block) diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 608545baf74..816ede4136a 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -5,6 +5,10 @@ module Gitlab extend self PathTraversalAttackError ||= Class.new(StandardError) + private_class_method def logger + @logger ||= Gitlab::AppLogger + end + # Ensure that the relative path will not traverse outside the base directory # We url decode the path to avoid passing invalid paths forward in url encoded format. # Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580 @@ -16,6 +20,7 @@ module Gitlab path_regex = %r{(\A(\.{1,2})\z|\A\.\.[/\\]|[/\\]\.\.\z|[/\\]\.\.[/\\]|\n)} if path.match?(path_regex) + logger.warn(message: "Potential path traversal attempt detected", path: "#{path}") raise PathTraversalAttackError, 'Invalid path' end @@ -37,6 +42,13 @@ module Gitlab raise StandardError, "path #{path} is not allowed" end + def check_allowed_absolute_path_and_path_traversal!(path, path_allowlist) + traversal_path = check_path_traversal!(path) + raise StandardError, "path is not a string!" unless traversal_path.is_a?(String) + + check_allowed_absolute_path!(traversal_path, path_allowlist) + end + def decode_path(encoded_path) decoded = CGI.unescape(encoded_path) if decoded != CGI.unescape(decoded) diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index 255fa0169bf..3c954f817a7 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -22,10 +22,12 @@ module Gitlab # end # def strong_memoize(name) - if strong_memoized?(name) - instance_variable_get(ivar(name)) + key = ivar(name) + + if instance_variable_defined?(key) + instance_variable_get(key) else - instance_variable_set(ivar(name), yield) + instance_variable_set(key, yield) end end @@ -34,13 +36,23 @@ module Gitlab end def clear_memoization(name) - remove_instance_variable(ivar(name)) if instance_variable_defined?(ivar(name)) + key = ivar(name) + remove_instance_variable(key) if instance_variable_defined?(key) end private + # Convert `"name"`/`:name` into `:@name` + # + # Depending on a type ensure that there's a single memory allocation def ivar(name) - "@#{name}" + if name.is_a?(Symbol) + name.to_s.prepend("@").to_sym + elsif name.is_a?(String) + :"@#{name}" + else + raise ArgumentError, "Invalid type of '#{name}'" + end end end end diff --git a/lib/gitlab/wiki_pages/front_matter_parser.rb b/lib/gitlab/wiki_pages/front_matter_parser.rb index 0ceec39782c..ee30fa907f4 100644 --- a/lib/gitlab/wiki_pages/front_matter_parser.rb +++ b/lib/gitlab/wiki_pages/front_matter_parser.rb @@ -3,8 +3,6 @@ module Gitlab module WikiPages class FrontMatterParser - FEATURE_FLAG = :wiki_front_matter - # We limit the maximum length of text we are prepared to parse as YAML, to # avoid exploitations and attempts to consume memory and CPU. We allow for: # - a title line @@ -30,18 +28,12 @@ module Gitlab end # @param [String] wiki_content - # @param [FeatureGate] feature_gate The scope for feature availability - def initialize(wiki_content, feature_gate) + def initialize(wiki_content) @wiki_content = wiki_content - @feature_gate = feature_gate - end - - def self.enabled?(gate = nil) - Feature.enabled?(FEATURE_FLAG, gate) end def parse - return empty_result unless enabled? && wiki_content.present? + return empty_result unless wiki_content.present? return empty_result(block.error) unless block.valid? Result.new(front_matter: block.data, content: strip_front_matter_block) @@ -94,22 +86,18 @@ module Gitlab private - attr_reader :wiki_content, :feature_gate + attr_reader :wiki_content def empty_result(reason = nil, error = nil) Result.new(content: wiki_content, reason: reason, error: error) end - def enabled? - self.class.enabled?(feature_gate) - end - def block @block ||= parse_front_matter_block end def parse_front_matter_block - wiki_content.match(Gitlab::FrontMatter::PATTERN) { |m| Block.new(*m.captures) } || Block.new + wiki_content.match(Gitlab::FrontMatter::PATTERN) { |m| Block.new(m[:delim], m[:lang], m[:front_matter]) } || Block.new end def strip_front_matter_block |