diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-04-21 02:50:22 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-04-21 02:50:22 +0300 |
| commit | 9dc93a4519d9d5d7be48ff274127136236a3adb3 (patch) | |
| tree | 70467ae3692a0e35e5ea56bcb803eb512a10bedb /lib/gitlab | |
| parent | 4b0f34b6d759d6299322b3a54453e930c6121ff0 (diff) | |
Add latest changes from gitlab-org/gitlab@13-11-stable-eev13.11.0-rc43
Diffstat (limited to 'lib/gitlab')
287 files changed, 4353 insertions, 1780 deletions
diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index c8b8d6c259d..786c5bf675b 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -132,7 +132,7 @@ module Gitlab EnvironmentsFinder .new(project, nil, { name: environment_name }) - .find + .execute .first end end diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index 178ebe0d4d4..b4752ed9e5b 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -31,14 +31,34 @@ module Gitlab @params = params @sort = params[:sort] || :end_event @direction = params[:direction] || :desc + @page = params[:page] || 1 + @per_page = MAX_RECORDS end + # rubocop: disable CodeReuse/ActiveRecord def serialized_records strong_memoize(:serialized_records) do # special case (legacy): 'Test' and 'Staging' stages should show Ci::Build records if default_test_stage? || default_staging_stage? + ci_build_join = mr_metrics_table + .join(build_table) + .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) + .join_sources + + records = ordered_and_limited_query + .joins(ci_build_join) + .select(build_table[:id], *time_columns) + + yield records if block_given? + ci_build_records = preload_ci_build_associations(records) + AnalyticsBuildSerializer.new.represent(ci_build_records.map { |e| e['build'] }) else + records = ordered_and_limited_query.select(*columns, *time_columns) + + yield records if block_given? + records = preload_associations(records) + records.map do |record| project = record.project attributes = record.attributes.merge({ @@ -51,10 +71,11 @@ module Gitlab end end end + # rubocop: enable CodeReuse/ActiveRecord private - attr_reader :stage, :query, :params, :sort, :direction + attr_reader :stage, :query, :params, :sort, :direction, :page, :per_page def columns MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name| @@ -74,41 +95,32 @@ module Gitlab MAPPINGS.fetch(subject_class).fetch(:serializer_class).new end - # Loading Ci::Build records instead of MergeRequest records # rubocop: disable CodeReuse/ActiveRecord - def ci_build_records - ci_build_join = mr_metrics_table - .join(build_table) - .on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) - .join_sources - - q = ordered_and_limited_query - .joins(ci_build_join) - .select(build_table[:id], *time_columns) - - results = execute_query(q).to_a + def preload_ci_build_associations(records) + results = records.map(&:attributes) Gitlab::CycleAnalytics::Updater.update!(results, from: 'id', to: 'build', klass: ::Ci::Build.includes({ project: [:namespace], user: [], pipeline: [] })) end + # rubocop: enable CodeReuse/ActiveRecord def ordered_and_limited_query - order_by(query, sort, direction, columns).limit(MAX_RECORDS) + strong_memoize(:ordered_and_limited_query) do + order_by(query, sort, direction, columns).page(page).per(per_page).without_count + end end - def records - results = ordered_and_limited_query - .select(*columns, *time_columns) - + # rubocop: disable CodeReuse/ActiveRecord + def preload_associations(records) # using preloader instead of includes to avoid AR generating a large column list ActiveRecord::Associations::Preloader.new.preload( - results, + records, MAPPINGS.fetch(subject_class).fetch(:includes_for_query) ) - results + records end - # rubocop: enable CodeReuse/ActiveRecord + # rubocop: enable CodeReuse/ActiveRecord def time_columns [ stage.start_event.timestamp_projection.as('start_event_timestamp'), diff --git a/lib/gitlab/analytics/unique_visits.rb b/lib/gitlab/analytics/unique_visits.rb index e367d33d743..723486231b1 100644 --- a/lib/gitlab/analytics/unique_visits.rb +++ b/lib/gitlab/analytics/unique_visits.rb @@ -3,8 +3,8 @@ module Gitlab module Analytics class UniqueVisits - def track_visit(visitor_id, target_id, time = Time.zone.now) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(target_id, values: visitor_id, time: time) + def track_visit(*args, **kwargs) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(*args, **kwargs) end # Returns number of unique visitors for given targets in given time frame diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index a75da3a682b..ceda82cb6f6 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -8,6 +8,9 @@ module Gitlab Attribute = Struct.new(:name, :type) + LOG_KEY = Labkit::Context::LOG_KEY + KNOWN_KEYS = Labkit::Context::KNOWN_KEYS + APPLICATION_ATTRIBUTES = [ Attribute.new(:project, Project), Attribute.new(:namespace, Namespace), @@ -24,6 +27,10 @@ module Gitlab application_context.use(&block) end + def self.with_raw_context(attributes = {}, &block) + Labkit::Context.with_context(attributes, &block) + end + def self.push(args) application_context = new(**args) Labkit::Context.push(application_context.to_lazy_hash) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 4c6254c9e69..6f6ac79c16b 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -24,9 +24,9 @@ module Gitlab PRIVATE_TOKEN_HEADER = 'HTTP_PRIVATE_TOKEN' PRIVATE_TOKEN_PARAM = :private_token - JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze + JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN' JOB_TOKEN_PARAM = :job_token - DEPLOY_TOKEN_HEADER = 'HTTP_DEPLOY_TOKEN'.freeze + DEPLOY_TOKEN_HEADER = 'HTTP_DEPLOY_TOKEN' RUNNER_TOKEN_PARAM = :token RUNNER_JOB_TOKEN_PARAM = :token diff --git a/lib/gitlab/auth/ldap/adapter.rb b/lib/gitlab/auth/ldap/adapter.rb index b7bb61f0677..7f85d3b1cd3 100644 --- a/lib/gitlab/auth/ldap/adapter.rb +++ b/lib/gitlab/auth/ldap/adapter.rb @@ -5,7 +5,7 @@ module Gitlab module Ldap class Adapter SEARCH_RETRY_FACTOR = [1, 1, 2, 3].freeze - MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size.freeze + MAX_SEARCH_RETRIES = Rails.env.test? ? 1 : SEARCH_RETRY_FACTOR.size attr_reader :provider, :ldap diff --git a/lib/gitlab/auth/saml/origin_validator.rb b/lib/gitlab/auth/saml/origin_validator.rb index 4ecc688888f..ff0d25314f7 100644 --- a/lib/gitlab/auth/saml/origin_validator.rb +++ b/lib/gitlab/auth/saml/origin_validator.rb @@ -4,7 +4,7 @@ module Gitlab module Auth module Saml class OriginValidator - AUTH_REQUEST_SESSION_KEY = "last_authn_request_id".freeze + AUTH_REQUEST_SESSION_KEY = "last_authn_request_id" def initialize(session) @session = session || {} diff --git a/lib/gitlab/background_migration/backfill_design_internal_ids.rb b/lib/gitlab/background_migration/backfill_design_internal_ids.rb index 553571d5d00..6d1df95c66d 100644 --- a/lib/gitlab/background_migration/backfill_design_internal_ids.rb +++ b/lib/gitlab/background_migration/backfill_design_internal_ids.rb @@ -97,13 +97,13 @@ module Gitlab ActiveRecord::Base.connection.execute <<~SQL WITH - starting_iids(project_id, iid) as ( + starting_iids(project_id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}( SELECT project_id, MAX(COALESCE(iid, 0)) FROM #{table} WHERE project_id BETWEEN #{start_id} AND #{end_id} GROUP BY project_id ), - with_calculated_iid(id, iid) as ( + with_calculated_iid(id, iid) as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported}( SELECT design.id, init.iid + ROW_NUMBER() OVER (PARTITION BY design.project_id ORDER BY design.id ASC) FROM #{table} as design, starting_iids as init diff --git a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb index 7484027a0fa..030dfd2d99b 100644 --- a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb +++ b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb @@ -8,7 +8,7 @@ module Gitlab updated_repository_storages = Projects::RepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id) Project.connection.execute <<-SQL - WITH repository_storage_cte as ( + WITH repository_storage_cte as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{updated_repository_storages.to_sql} ) UPDATE projects diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index 60682bd2ec1..b89ea7dc250 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -34,12 +34,18 @@ module Gitlab parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| - sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") + batch_metrics.time_operation(:update_all) do + sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") + end sleep(PAUSE_SECONDS) end end + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + private def connection diff --git a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb index 6014ccc12eb..691bdb457d7 100644 --- a/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb +++ b/lib/gitlab/background_migration/copy_merge_request_target_project_to_merge_request_metrics.rb @@ -8,7 +8,7 @@ module Gitlab def perform(start_id, stop_id) ActiveRecord::Base.connection.execute <<~SQL - WITH merge_requests_batch AS ( + WITH merge_requests_batch AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT id, target_project_id FROM merge_requests WHERE id BETWEEN #{Integer(start_id)} AND #{Integer(stop_id)} ) diff --git a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb index 68665db522e..83c01afa432 100644 --- a/lib/gitlab/background_migration/fix_projects_without_project_feature.rb +++ b/lib/gitlab/background_migration/fix_projects_without_project_feature.rb @@ -22,7 +22,7 @@ module Gitlab def sql(from_id, to_id) <<~SQL - WITH created_records AS ( + WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( INSERT INTO project_features ( project_id, merge_requests_access_level, diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb index e750b8ca374..b8e4562b3bf 100644 --- a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb +++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb @@ -136,7 +136,7 @@ module Gitlab # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT def create_sql(from_id, to_id) <<~SQL - WITH created_records AS ( + WITH created_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( INSERT INTO services (project_id, #{DEFAULTS.keys.map { |key| %("#{key}")}.join(',')}, created_at, updated_at) #{select_insert_values_sql(from_id, to_id)} RETURNING * @@ -149,7 +149,7 @@ module Gitlab # there is no uniq constraint on project_id and type pair, which prevents us from using ON CONFLICT def update_sql(from_id, to_id) <<~SQL - WITH updated_records AS ( + WITH updated_records AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( UPDATE services SET active = TRUE WHERE services.project_id BETWEEN #{Integer(from_id)} AND #{Integer(to_id)} AND services.properties = '{}' AND services.type = '#{Migratable::PrometheusService.type}' AND #{group_cluster_condition(from_id, to_id)} AND services.active = FALSE diff --git a/lib/gitlab/background_migration/fix_user_namespace_names.rb b/lib/gitlab/background_migration/fix_user_namespace_names.rb index d767cbfd8f5..cd5b4ab103d 100644 --- a/lib/gitlab/background_migration/fix_user_namespace_names.rb +++ b/lib/gitlab/background_migration/fix_user_namespace_names.rb @@ -14,7 +14,7 @@ module Gitlab def fix_namespace_names(from_id, to_id) ActiveRecord::Base.connection.execute <<~UPDATE_NAMESPACES - WITH namespaces_to_update AS ( + WITH namespaces_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT namespaces.id, users.name AS correct_name @@ -39,7 +39,7 @@ module Gitlab def fix_namespace_route_names(from_id, to_id) ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE - WITH routes_to_update AS ( + WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT routes.id, users.name AS correct_name diff --git a/lib/gitlab/background_migration/fix_user_project_route_names.rb b/lib/gitlab/background_migration/fix_user_project_route_names.rb index 6b99685fd68..e534f2449aa 100644 --- a/lib/gitlab/background_migration/fix_user_project_route_names.rb +++ b/lib/gitlab/background_migration/fix_user_project_route_names.rb @@ -8,7 +8,7 @@ module Gitlab class FixUserProjectRouteNames def perform(from_id, to_id) ActiveRecord::Base.connection.execute <<~ROUTES_UPDATE - WITH routes_to_update AS ( + WITH routes_to_update AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT routes.id, users.name || ' / ' || projects.name AS correct_name diff --git a/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb new file mode 100644 index 00000000000..b7a912da060 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_pages_to_zip_storage.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # migrates pages from legacy storage to zip format + # we intentionally use application code here because + # it has a lot of dependencies including models, carrierwave uploaders and service objects + # and copying all or part of this code in the background migration doesn't add much value + # see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54578 for discussion + class MigratePagesToZipStorage + def perform(start_id, stop_id) + ::Pages::MigrateFromLegacyStorageService.new(Gitlab::AppLogger, + ignore_invalid_entries: false, + mark_projects_as_not_deployed: false) + .execute_for_batch(start_id..stop_id) + end + end + end +end diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb index 4eaef26c9c6..9ecf53317d0 100644 --- a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb +++ b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb @@ -6,7 +6,7 @@ module Gitlab # project_features.container_registry_access_level for the projects within # the given range of ids. class MoveContainerRegistryEnabledToProjectFeature - MAX_BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 300 module Migratable # Migration model namespace isolated from application code. diff --git a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb index 78140b768fc..28ff2070209 100644 --- a/lib/gitlab/background_migration/populate_has_vulnerabilities.rb +++ b/lib/gitlab/background_migration/populate_has_vulnerabilities.rb @@ -8,21 +8,23 @@ module Gitlab class ProjectSetting < ActiveRecord::Base # rubocop:disable Style/Documentation self.table_name = 'project_settings' - UPSERT_SQL = <<~SQL - WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS ( - SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids}) - ) - INSERT INTO project_settings - (project_id, has_vulnerabilities, created_at, updated_at) - (SELECT * FROM upsert_data) - ON CONFLICT (project_id) - DO UPDATE SET - has_vulnerabilities = true, - updated_at = EXCLUDED.updated_at - SQL - def self.upsert_for(project_ids) - connection.execute(UPSERT_SQL % { project_ids: project_ids.join(', ') }) + connection.execute(upsert_sql % { project_ids: project_ids.join(', ') }) + end + + def self.upsert_sql + <<~SQL + WITH upsert_data (project_id, has_vulnerabilities, created_at, updated_at) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + SELECT projects.id, true, current_timestamp, current_timestamp FROM projects WHERE projects.id IN (%{project_ids}) + ) + INSERT INTO project_settings + (project_id, has_vulnerabilities, created_at, updated_at) + (SELECT * FROM upsert_data) + ON CONFLICT (project_id) + DO UPDATE SET + has_vulnerabilities = true, + updated_at = EXCLUDED.updated_at + SQL end end diff --git a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb index eb4bc0aaf28..28cc4a5e3fa 100644 --- a/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb +++ b/lib/gitlab/background_migration/populate_merge_request_assignees_table.rb @@ -11,7 +11,7 @@ module Gitlab MergeRequest .where(merge_request_assignees_not_exists_clause) .where(id: from_id..to_id) - .where('assignee_id IS NOT NULL') + .where.not(assignee_id: nil) .select(:id, :assignee_id) .to_sql diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index 7b18e617c81..888a12f2330 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -32,7 +32,7 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid }.freeze NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze - PACK_PATTERN = "NnnnnN".freeze + PACK_PATTERN = "NnnnnN" def self.call(value) Digest::UUID.uuid_v5(namespace_id, value) diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb new file mode 100644 index 00000000000..5930d65bc2c --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_project_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + module ReferenceParser + # isolated Banzai::ReferenceParser::MentionedGroupParser + class IsolatedMentionedProjectParser < ::Banzai::ReferenceParser::MentionedProjectParser + extend ::Gitlab::Utils::Override + + self.reference_type = :user + + override :references_relation + def references_relation + ::Gitlab::BackgroundMigration::UserMentions::Models::Project + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb new file mode 100644 index 00000000000..f5f98517433 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/banzai/reference_parser/isolated_mentioned_user_parser.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Banzai + module ReferenceParser + # isolated Banzai::ReferenceParser::MentionedGroupParser + class IsolatedMentionedUserParser < ::Banzai::ReferenceParser::MentionedUserParser + extend ::Gitlab::Utils::Override + + self.reference_type = :user + + override :references_relation + def references_relation + ::Gitlab::BackgroundMigration::UserMentions::Models::User + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb index 1d3a3af81a1..8610129533d 100644 --- a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb +++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_reference_extractor.rb @@ -7,7 +7,7 @@ module Gitlab module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class IsolatedReferenceExtractor < ::Gitlab::ReferenceExtractor - REFERABLES = %i(isolated_mentioned_group).freeze + REFERABLES = %i(isolated_mentioned_group isolated_mentioned_user isolated_mentioned_project).freeze REFERABLES.each do |type| define_method("#{type}s") do diff --git a/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb new file mode 100644 index 00000000000..0334ea1dd08 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/lib/gitlab/isolated_visibility_level.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Lib + module Gitlab + # Gitlab::IsolatedVisibilityLevel module + # + # Define allowed public modes that can be used for + # GitLab projects to determine project public mode + # + module IsolatedVisibilityLevel + extend ::ActiveSupport::Concern + + included do + scope :public_to_user, -> (user = nil) do + where(visibility_level: IsolatedVisibilityLevel.levels_for_user(user)) + end + end + + PRIVATE = 0 unless const_defined?(:PRIVATE) + INTERNAL = 10 unless const_defined?(:INTERNAL) + PUBLIC = 20 unless const_defined?(:PUBLIC) + + class << self + def levels_for_user(user = nil) + return [PUBLIC] unless user + + if user.can_read_all_resources? + [PRIVATE, INTERNAL, PUBLIC] + elsif user.external? + [PUBLIC] + else + [INTERNAL, PUBLIC] + end + end + end + + def private? + visibility_level_value == PRIVATE + end + + def internal? + visibility_level_value == INTERNAL + end + + def public? + visibility_level_value == PUBLIC + end + + def visibility_level_value + self[visibility_level_field] + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb index bdb4d6c7d48..f4cc96c8bc0 100644 --- a/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/commit_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class CommitUserMention < ActiveRecord::Base self.table_name = 'commit_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :commit_id diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb new file mode 100644 index 00000000000..ba6b783f9f1 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_feature_gate.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + module Concerns + # isolated FeatureGate module + module IsolatedFeatureGate + def flipper_id + return if new_record? + + "#{self.class.name}:#{id}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb index be9c0ad2b3a..f684f789ea9 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/isolated_mentionable.rb @@ -70,8 +70,8 @@ module Gitlab def build_mention_values(resource_foreign_key) refs = all_references(author) - mentioned_users_ids = array_to_sql(refs.mentioned_users.pluck(:id)) - mentioned_projects_ids = array_to_sql(refs.mentioned_projects.pluck(:id)) + mentioned_users_ids = array_to_sql(refs.isolated_mentioned_users.pluck(:id)) + mentioned_projects_ids = array_to_sql(refs.isolated_mentioned_projects.pluck(:id)) mentioned_groups_ids = array_to_sql(refs.isolated_mentioned_groups.pluck(:id)) return if mentioned_users_ids.blank? && mentioned_projects_ids.blank? && mentioned_groups_ids.blank? diff --git a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb index 5cadfa45b5b..75759ed0111 100644 --- a/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb +++ b/lib/gitlab/background_migration/user_mentions/models/concerns/namespace/recursive_traversal.rb @@ -6,7 +6,7 @@ module Gitlab module Models module Concerns module Namespace - # extracted methods for recursive traversing of namespace hierarchy + # isolate recursive traversal code for namespace hierarchy module RecursiveTraversal extend ActiveSupport::Concern diff --git a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb index bdb90b5d2b9..d010d68600d 100644 --- a/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb +++ b/lib/gitlab/background_migration/user_mentions/models/design_management/design.rb @@ -10,6 +10,9 @@ module Gitlab include EachBatch include Concerns::MentionableMigrationMethods + self.table_name = 'design_management_designs' + self.inheritance_column = :_type_disabled + def self.user_mention_model Gitlab::BackgroundMigration::UserMentions::Models::DesignUserMention end diff --git a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb index 68205ecd3c2..eb00f6cfa3f 100644 --- a/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/design_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class DesignUserMention < ActiveRecord::Base self.table_name = 'design_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :design_id diff --git a/lib/gitlab/background_migration/user_mentions/models/epic.rb b/lib/gitlab/background_migration/user_mentions/models/epic.rb index 61d9244a4c9..cfd9a4faa9b 100644 --- a/lib/gitlab/background_migration/user_mentions/models/epic.rb +++ b/lib/gitlab/background_migration/user_mentions/models/epic.rb @@ -17,10 +17,10 @@ module Gitlab cache_markdown_field :description, issuable_state_filter_enabled: true self.table_name = 'epics' + self.inheritance_column = :_type_disabled - belongs_to :author, class_name: "User" - belongs_to :project - belongs_to :group + belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" + belongs_to :group, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group" def self.user_mention_model Gitlab::BackgroundMigration::UserMentions::Models::EpicUserMention diff --git a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb index 4e3ce9bf3a7..579e4d99612 100644 --- a/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class EpicUserMention < ActiveRecord::Base self.table_name = 'epic_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :epic_id diff --git a/lib/gitlab/background_migration/user_mentions/models/group.rb b/lib/gitlab/background_migration/user_mentions/models/group.rb index bc04172b9a2..a8b4b59b06c 100644 --- a/lib/gitlab/background_migration/user_mentions/models/group.rb +++ b/lib/gitlab/background_migration/user_mentions/models/group.rb @@ -7,6 +7,8 @@ module Gitlab # isolated Group model class Group < ::Gitlab::BackgroundMigration::UserMentions::Models::Namespace self.store_full_sti_class = false + self.inheritance_column = :_type_disabled + has_one :saml_provider def self.declarative_policy_class diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb index 6b52afea17c..13addcc3c55 100644 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request.rb +++ b/lib/gitlab/background_migration/user_mentions/models/merge_request.rb @@ -17,10 +17,11 @@ module Gitlab cache_markdown_field :description, issuable_state_filter_enabled: true self.table_name = 'merge_requests' + self.inheritance_column = :_type_disabled - belongs_to :author, class_name: "User" - belongs_to :target_project, class_name: "Project" - belongs_to :source_project, class_name: "Project" + belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" + belongs_to :target_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" + belongs_to :source_project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" alias_attribute :project, :target_project diff --git a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb index e9b85e9cb8c..4a85892d7b8 100644 --- a/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/models/merge_request_user_mention.rb @@ -7,6 +7,7 @@ module Gitlab module Models class MergeRequestUserMention < ActiveRecord::Base self.table_name = 'merge_request_user_mentions' + self.inheritance_column = :_type_disabled def self.resource_foreign_key :merge_request_id diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb index 8fa0db5fd4b..a2b50c41f4a 100644 --- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -5,9 +5,11 @@ module Gitlab module UserMentions module Models # isolated Namespace model - class Namespace < ApplicationRecord - include FeatureGate - include ::Gitlab::VisibilityLevel + class Namespace < ActiveRecord::Base + self.inheritance_column = :_type_disabled + + include Concerns::IsolatedFeatureGate + include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel include ::Gitlab::Utils::StrongMemoize include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal @@ -21,8 +23,13 @@ module Gitlab parent_id.present? || parent.present? end + # Deprecated, use #licensed_feature_available? instead. Remove once Namespace#feature_available? isn't used anymore. + def feature_available?(feature) + licensed_feature_available?(feature) + end + # Overridden in EE::Namespace - def feature_available?(_feature) + def licensed_feature_available?(_feature) false end end diff --git a/lib/gitlab/background_migration/user_mentions/models/note.rb b/lib/gitlab/background_migration/user_mentions/models/note.rb index a3224c8c456..7da933c7b11 100644 --- a/lib/gitlab/background_migration/user_mentions/models/note.rb +++ b/lib/gitlab/background_migration/user_mentions/models/note.rb @@ -16,9 +16,9 @@ module Gitlab attr_mentionable :note, pipeline: :note cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true - belongs_to :author, class_name: "User" + belongs_to :author, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::User" belongs_to :noteable, polymorphic: true - belongs_to :project + belongs_to :project, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Project" def for_personal_snippet? noteable && noteable.class.name == 'PersonalSnippet' diff --git a/lib/gitlab/background_migration/user_mentions/models/project.rb b/lib/gitlab/background_migration/user_mentions/models/project.rb new file mode 100644 index 00000000000..4e02bf97d12 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/project.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Namespace model + class Project < ActiveRecord::Base + include Concerns::IsolatedFeatureGate + include Gitlab::BackgroundMigration::UserMentions::Lib::Gitlab::IsolatedVisibilityLevel + + self.table_name = 'projects' + self.inheritance_column = :_type_disabled + + belongs_to :group, -> { where(type: 'Group') }, foreign_key: 'namespace_id', class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Group" + belongs_to :namespace, class_name: "::Gitlab::BackgroundMigration::UserMentions::Models::Namespace" + alias_method :parent, :namespace + + # Returns a collection of projects that is either public or visible to the + # logged in user. + def self.public_or_visible_to_user(user = nil, min_access_level = nil) + min_access_level = nil if user&.can_read_all_resources? + + return public_to_user unless user + + if user.is_a?(::Gitlab::BackgroundMigration::UserMentions::Models::User) + where('EXISTS (?) OR projects.visibility_level IN (?)', + user.authorizations_for_projects(min_access_level: min_access_level), + levels_for_user(user)) + end + end + + def grafana_integration + nil + end + + def default_issues_tracker? + true # we do not care of the issue tracker type(internal or external) when parsing mentions + end + + def visibility_level_field + :visibility_level + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/user.rb b/lib/gitlab/background_migration/user_mentions/models/user.rb new file mode 100644 index 00000000000..a30220b6934 --- /dev/null +++ b/lib/gitlab/background_migration/user_mentions/models/user.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module UserMentions + module Models + # isolated Namespace model + class User < ActiveRecord::Base + include Concerns::IsolatedFeatureGate + + self.table_name = 'users' + self.inheritance_column = :_type_disabled + + has_many :project_authorizations, dependent: :delete_all # rubocop:disable Cop/ActiveRecordDependent + + def authorizations_for_projects(min_access_level: nil, related_project_column: 'projects.id') + authorizations = project_authorizations + .select(1) + .where("project_authorizations.project_id = #{related_project_column}") + + return authorizations unless min_access_level.present? + + authorizations.where('project_authorizations.access_level >= ?', min_access_level) + end + + def can_read_all_resources? + can?(:read_all_resources) + end + + def can?(action, subject = :global) + Ability.allowed?(self, action, subject) + end + end + end + end + end +end diff --git a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb index baacc912df3..665ad7abcbb 100644 --- a/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb +++ b/lib/gitlab/background_migration/wrongfully_confirmed_email_unconfirmer.rb @@ -27,7 +27,7 @@ module Gitlab joins(:user) .merge(UserModel.active) .where(id: (start_id..stop_id)) - .where('emails.confirmed_at IS NOT NULL') + .where.not('emails.confirmed_at' => nil) .where('emails.confirmed_at = users.confirmed_at') .where('emails.email <> users.email') .where('NOT EXISTS (SELECT 1 FROM user_synced_attributes_metadata WHERE user_id=users.id AND email_synced IS true)') @@ -57,7 +57,7 @@ module Gitlab def update_email_records(start_id, stop_id) EmailModel.connection.execute <<-SQL - WITH md5_strings as ( + WITH md5_strings as #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( #{email_query_for_update(start_id, stop_id).to_sql} ) UPDATE #{EmailModel.connection.quote_table_name(EmailModel.table_name)} diff --git a/lib/gitlab/batch_pop_queueing.rb b/lib/gitlab/batch_pop_queueing.rb index e18f1320ea4..62fc8cd048e 100644 --- a/lib/gitlab/batch_pop_queueing.rb +++ b/lib/gitlab/batch_pop_queueing.rb @@ -46,7 +46,8 @@ module Gitlab def initialize(namespace, queue_id) raise ArgumentError if namespace.empty? || queue_id.empty? - @namespace, @queue_id = namespace, queue_id + @namespace = namespace + @queue_id = queue_id end ## diff --git a/lib/gitlab/bullet.rb b/lib/gitlab/bullet.rb new file mode 100644 index 00000000000..f5f8a316855 --- /dev/null +++ b/lib/gitlab/bullet.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module Bullet + extend self + + def enabled? + Gitlab::Utils.to_boolean(ENV['ENABLE_BULLET'], default: false) + end + alias_method :extra_logging_enabled?, :enabled? + + def configure_bullet? + defined?(::Bullet) && (enabled? || Rails.env.development?) + end + end +end diff --git a/lib/gitlab/bullet/exclusions.rb b/lib/gitlab/bullet/exclusions.rb new file mode 100644 index 00000000000..f897ff492d9 --- /dev/null +++ b/lib/gitlab/bullet/exclusions.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Bullet + class Exclusions + def initialize(config_file = Gitlab.root.join('config/bullet.yml')) + @config_file = config_file + end + + def execute + exclusions.map { |v| v['exclude'] } + end + + def validate_paths! + exclusions.each do |properties| + next unless properties['path_with_method'] + + file = properties['exclude'].first + + raise "Bullet: File used by #{config_file} doesn't exist, validate the #{file} exclusion!" unless File.exist?(file) + end + end + + private + + attr_reader :config_file + + def exclusions + @exclusions ||= if File.exist?(config_file) + YAML.load_file(config_file)['exclusions']&.values || [] + else + [] + end + end + end + end +end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index d981f263c5e..9e958eb52fb 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -69,7 +69,9 @@ module Gitlab def load_from_project return unless commit - self.sha, self.status, self.ref = commit.sha, commit.status, project.default_branch + self.sha = commit.sha + self.status = commit.status + self.ref = project.default_branch end # We only cache the status for the HEAD commit of a project diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb index 105050936ce..be8009750da 100644 --- a/lib/gitlab/changelog/config.rb +++ b/lib/gitlab/changelog/config.rb @@ -17,7 +17,24 @@ module Gitlab # The default template to use for generating release sections. DEFAULT_TEMPLATE = File.read(File.join(__dir__, 'template.tpl')) - attr_accessor :date_format, :categories, :template + # The regex to use for extracting the version from a Git tag. + # + # This regex is based on the official semantic versioning regex (as found + # on https://semver.org/), with the addition of allowing a "v" at the + # start of a tag name. + # + # We default to a strict regex as we simply don't know what kind of data + # users put in their tags. As such, using simpler patterns (e.g. just + # `\d+` for the major version) could lead to unexpected results. + # + # We use a String here as `Gitlab::UntrustedRegexp` is a mutable object. + DEFAULT_TAG_REGEX = '^v?(?P<major>0|[1-9]\d*)' \ + '\.(?P<minor>0|[1-9]\d*)' \ + '\.(?P<patch>0|[1-9]\d*)' \ + '(?:-(?P<pre>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))' \ + '?(?:\+(?P<meta>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$' + + attr_accessor :date_format, :categories, :template, :tag_regex def self.from_git(project) if (yaml = project.repository.changelog_config) @@ -46,6 +63,10 @@ module Gitlab end end + if (regex = hash['tag_regex']) + config.tag_regex = regex + end + config end @@ -54,6 +75,7 @@ module Gitlab @date_format = DEFAULT_DATE_FORMAT @template = Parser.new.parse_and_transform(DEFAULT_TEMPLATE) @categories = {} + @tag_regex = DEFAULT_TAG_REGEX end def contributor?(user) diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb index 029a9210dc9..495f12882e5 100644 --- a/lib/gitlab/chaos.rb +++ b/lib/gitlab/chaos.rb @@ -43,9 +43,9 @@ module Gitlab Kernel.sleep(duration_s) end - # Kill will send a SIGKILL signal to the current process - def self.kill - Process.kill("KILL", Process.pid) + # Kill will send the given signal to the current process. + def self.kill(signal) + Process.kill(signal, Process.pid) end def self.run_gc diff --git a/lib/gitlab/ci/build/artifacts/metadata.rb b/lib/gitlab/ci/build/artifacts/metadata.rb index c5afb16ab1a..88d624503df 100644 --- a/lib/gitlab/ci/build/artifacts/metadata.rb +++ b/lib/gitlab/ci/build/artifacts/metadata.rb @@ -17,7 +17,9 @@ module Gitlab attr_reader :stream, :path, :full_version def initialize(stream, path, **opts) - @stream, @path, @opts = stream, path, opts + @stream = stream + @path = path + @opts = opts @full_version = read_version end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index d3f030c3b36..23b0c93a3ee 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -17,12 +17,14 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root + attr_reader :root, :context, :ref - def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil) + def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil) @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline) @context.set_deadline(TIMEOUT_SECONDS) + @ref = ref + @config = expand_config(config) @root = Entry::Root.new(@config) @@ -94,9 +96,7 @@ module Gitlab initial_config = Config::External::Processor.new(initial_config, @context).perform initial_config = Config::Extendable.new(initial_config).to_hash initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash - initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash - - initial_config + Config::EdgeStagesInjector.new(initial_config).to_hash end def find_sha(project) diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index cf599ce5294..f9688c500d2 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -8,8 +8,8 @@ module Gitlab # Entry that represents a cache configuration # class Cache < ::Gitlab::Config::Entry::Simplifiable - strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job) } - strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job) } + strategy :Caches, if: -> (config) { Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) } + strategy :Cache, if: -> (config) { Feature.disabled?(:multiple_cache_per_job, default_enabled: :yaml) } class Caches < ::Gitlab::Config::Entry::ComposableArray include ::Gitlab::Config::Entry::Validatable @@ -17,8 +17,6 @@ module Gitlab MULTIPLE_CACHE_LIMIT = 4 validations do - validates :config, presence: true - validate do unless config.is_a?(Hash) || config.is_a?(Array) errors.add(:config, 'can only be a Hash or an Array') diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 9584d19bdec..947b6787aa0 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -124,7 +124,9 @@ module Gitlab stage: stage_value, extends: extends, rules: rules_value, - variables: root_and_job_variables_value, + variables: root_and_job_variables_value, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + job_variables: job_variables, + root_variables_inheritance: root_variables_inheritance, only: only_value, except: except_value, resource_group: resource_group }.compact @@ -139,6 +141,14 @@ module Gitlab root_variables.merge(variables_value.to_h) end + def job_variables + variables_value.to_h + end + + def root_variables_inheritance + inherit_entry&.variables_entry&.value + end + def manual_action? self.when == 'manual' end diff --git a/lib/gitlab/ci/config/entry/product/variables.rb b/lib/gitlab/ci/config/entry/product/variables.rb index aa34cfb3acc..e869e0bbb31 100644 --- a/lib/gitlab/ci/config/entry/product/variables.rb +++ b/lib/gitlab/ci/config/entry/product/variables.rb @@ -25,8 +25,7 @@ module Gitlab def value @config - .map { |key, value| [key.to_s, Array(value).map(&:to_s)] } - .to_h + .to_h { |key, value| [key.to_s, Array(value).map(&:to_s)] } end end end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index dc164d752be..efb469ee32a 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -18,7 +18,7 @@ module Gitlab end def value - Hash[@config.map { |key, value| [key.to_s, expand_value(value)[:value]] }] + @config.to_h { |key, value| [key.to_s, expand_value(value)[:value]] } end def self.default(**) @@ -26,7 +26,7 @@ module Gitlab end def value_with_data - Hash[@config.map { |key, value| [key.to_s, expand_value(value)] }] + @config.to_h { |key, value| [key.to_s, expand_value(value)] } end def use_value_data? diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index b85b7a9edeb..3216d4eaac4 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -34,6 +34,7 @@ module Gitlab .compact .map(&method(:normalize_location)) .flat_map(&method(:expand_project_files)) + .flat_map(&method(:expand_wildcard_paths)) .map(&method(:expand_variables)) .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) @@ -63,6 +64,17 @@ module Gitlab end end + def expand_wildcard_paths(location) + return location unless ::Feature.enabled?(:ci_wildcard_file_paths, context.project, default_enabled: :yaml) + + # We only support local files for wildcard paths + return location unless location[:local] && location[:local].include?('*') + + context.project.repository.search_files_by_wildcard_path(location[:local], context.sha).map do |path| + { local: path } + end + end + def normalize_location_string(location) if ::Gitlab::UrlSanitizer.valid?(location) { remote: location } diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb index 5a23836d8a0..5cabbc86d3e 100644 --- a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb +++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb @@ -43,9 +43,10 @@ module Gitlab { name: name, instance: instance, - variables: variables, + variables: variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + job_variables: variables, parallel: { total: total } - } + }.compact end def name diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index c811ef211d6..12e182b38fc 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -10,10 +10,6 @@ module Gitlab ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true) end - def self.instance_variables_ui_enabled? - ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true) - end - def self.pipeline_latest? ::Feature.enabled?(:ci_pipeline_latest, default_enabled: true) end @@ -60,16 +56,12 @@ module Gitlab ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: false) end - def self.display_codequality_backend_comparison?(project) - ::Feature.enabled?(:codequality_backend_comparison, project, default_enabled: :yaml) - end - def self.multiple_cache_per_job? ::Feature.enabled?(:multiple_cache_per_job, default_enabled: :yaml) end - def self.ci_commit_pipeline_mini_graph_vue_enabled?(project) - ::Feature.enabled?(:ci_commit_pipeline_mini_graph_vue, project, default_enabled: :yaml) + def self.gldropdown_tags_enabled? + ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml) end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index af06e124736..a6ae249fa58 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -72,16 +72,16 @@ module Gitlab def key @key ||= begin - key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true) - Gitlab::CurrentSettings.ci_jwt_signing_key - else - Rails.application.secrets.openid_connect_signing_key - end + key_data = if Feature.enabled?(:ci_jwt_signing_key, build.project, default_enabled: true) + Gitlab::CurrentSettings.ci_jwt_signing_key + else + Rails.application.secrets.openid_connect_signing_key + end - raise NoSigningKeyError unless key_data + raise NoSigningKeyError unless key_data - OpenSSL::PKey::RSA.new(key_data) - end + OpenSSL::PKey::RSA.new(key_data) + end end def public_key diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 815fe6bac6d..c3c1728602c 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -12,7 +12,7 @@ module Gitlab :seeds_block, :variables_attributes, :push_options, :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, # These attributes are set by Chains during processing: - :config_content, :yaml_processor_result, :pipeline_seed + :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed ) do include Gitlab::Utils::StrongMemoize @@ -84,7 +84,7 @@ module Gitlab end def metrics - @metrics ||= ::Gitlab::Ci::Pipeline::Metrics.new + @metrics ||= ::Gitlab::Ci::Pipeline::Metrics end def observe_creation_duration(duration) @@ -97,6 +97,11 @@ module Gitlab .observe({ source: pipeline.source.to_s }, pipeline.total_size) end + def increment_pipeline_failure_reason_counter(reason) + metrics.pipeline_failure_reason_counter + .increment(reason: (reason || :unknown_failure).to_s) + end + def dangling_build? %i[ondemand_dast_scan webide].include?(source) end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index c3fbd0c9e24..8f1c49563f2 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -14,6 +14,7 @@ module Gitlab result = ::Gitlab::Ci::YamlProcessor.new( @command.config_content, { project: project, + ref: @pipeline.ref, sha: @pipeline.sha, user: current_user, parent_pipeline: parent_pipeline diff --git a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb index 3c910963a2a..cceaa52de16 100644 --- a/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb +++ b/lib/gitlab/ci/pipeline/chain/evaluate_workflow_rules.rb @@ -9,6 +9,8 @@ module Gitlab include Chain::Helpers def perform! + @command.workflow_rules_result = workflow_rules_result + error('Pipeline filtered out by workflow rules.') unless workflow_passed? end @@ -19,27 +21,33 @@ module Gitlab private def workflow_passed? - strong_memoize(:workflow_passed) do - workflow_rules.evaluate(@pipeline, global_context).pass? + workflow_rules_result.pass? + end + + def workflow_rules_result + strong_memoize(:workflow_rules_result) do + workflow_rules.evaluate(@pipeline, global_context) end end def workflow_rules Gitlab::Ci::Build::Rules.new( - workflow_config[:rules], default_when: 'always') + workflow_rules_config, default_when: 'always') end def global_context Gitlab::Ci::Build::Context::Global.new( - @pipeline, yaml_variables: workflow_config[:yaml_variables]) + @pipeline, yaml_variables: @command.yaml_processor_result.root_variables) end def has_workflow_rules? - workflow_config[:rules].present? + workflow_rules_config.present? end - def workflow_config - @command.yaml_processor_result.workflow_attributes || {} + def workflow_rules_config + strong_memoize(:workflow_rules_config) do + @command.yaml_processor_result.workflow_rules + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index d7271df1694..9988b6f18ed 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -12,7 +12,8 @@ module Gitlab end pipeline.add_error_message(message) - pipeline.drop!(drop_reason) if drop_reason && persist_pipeline? + + drop_pipeline!(drop_reason) # TODO: consider not to rely on AR errors directly as they can be # polluted with other unrelated errors (e.g. state machine) @@ -24,8 +25,21 @@ module Gitlab pipeline.add_warning_message(message) end - def persist_pipeline? - command.save_incompleted && !pipeline.readonly? + private + + def drop_pipeline!(drop_reason) + return if pipeline.readonly? + + if drop_reason && command.save_incompleted + # Project iid must be called outside a transaction, so we ensure it is set here + # otherwise it may be set within the state transition transaction of the drop! call + # which it will lock the InternalId row for the whole transaction + pipeline.ensure_project_iid! + + pipeline.drop!(drop_reason) + else + command.increment_pipeline_failure_reason_counter(drop_reason) + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb index 0d7449813b4..b17ae77d445 100644 --- a/lib/gitlab/ci/pipeline/chain/metrics.rb +++ b/lib/gitlab/ci/pipeline/chain/metrics.rb @@ -14,7 +14,7 @@ module Gitlab end def counter - ::Gitlab::Ci::Pipeline::Metrics.new.pipelines_created_counter + ::Gitlab::Ci::Pipeline::Metrics.pipelines_created_counter end end end diff --git a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb index 1eb7474e915..c1b6dfb7e36 100644 --- a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb +++ b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb @@ -8,9 +8,7 @@ module Gitlab # After pipeline has been successfully created we can start processing it. class Process < Chain::Base def perform! - ::Ci::ProcessPipelineService - .new(@pipeline) - .execute + ::Ci::InitialPipelineProcessWorker.perform_async(pipeline.id) end def break? diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 7b537125b9b..66fc6741252 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -11,6 +11,10 @@ module Gitlab def perform! raise ArgumentError, 'missing YAML processor result' unless @command.yaml_processor_result + if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml) + raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result + end + # Allocate next IID. This operation must be outside of transactions of pipeline creations. pipeline.ensure_project_iid! pipeline.ensure_ci_ref! @@ -38,7 +42,21 @@ module Gitlab def pipeline_seed strong_memoize(:pipeline_seed) do stages_attributes = @command.yaml_processor_result.stages_attributes - Gitlab::Ci::Pipeline::Seed::Pipeline.new(pipeline, stages_attributes) + Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes) + end + end + + def context + Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: root_variables) + end + + def root_variables + if ::Feature.enabled?(:ci_workflow_rules_variables, pipeline.project, default_enabled: :yaml) + ::Gitlab::Ci::Variables::Helpers.merge_variables( + @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables + ) + else + @command.yaml_processor_result.root_variables end end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index d056501a6d3..6149d2f04d7 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -10,77 +10,116 @@ module Gitlab InvalidResponseCode = Class.new(StandardError) - VALIDATION_REQUEST_TIMEOUT = 5 + DEFAULT_VALIDATION_REQUEST_TIMEOUT = 5 + ACCEPTED_STATUS = 200 + DOT_COM_REJECTED_STATUS = 406 + GENERAL_REJECTED_STATUS = (400..499).freeze def perform! + return unless enabled? + pipeline_authorized = validate_external log_message = pipeline_authorized ? 'authorized' : 'not authorized' - Gitlab::AppLogger.info(message: "Pipeline #{log_message}", project_id: @pipeline.project.id, user_id: @pipeline.user.id) + Gitlab::AppLogger.info(message: "Pipeline #{log_message}", project_id: project.id, user_id: current_user.id) error('External validation failed', drop_reason: :external_validation_failure) unless pipeline_authorized end def break? - @pipeline.errors.any? + pipeline.errors.any? end private + def enabled? + return true unless Gitlab.com? + + ::Feature.enabled?(:ci_external_validation_service, project, default_enabled: :yaml) + end + def validate_external return true unless validation_service_url # 200 - accepted - # 4xx - not accepted + # 406 - not accepted on GitLab.com + # 4XX - not accepted for other installations # everything else - accepted and logged response_code = validate_service_request.code case response_code - when 200 + when ACCEPTED_STATUS true - when 400..499 + when rejected_status false else raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}" end rescue => ex - Gitlab::ErrorTracking.track_exception(ex) + Gitlab::ErrorTracking.track_exception(ex, project_id: project.id) true end + def rejected_status + if Gitlab.com? + DOT_COM_REJECTED_STATUS + else + GENERAL_REJECTED_STATUS + end + end + def validate_service_request + headers = { + 'X-Gitlab-Correlation-id' => Labkit::Correlation::CorrelationId.current_id, + 'X-Gitlab-Token' => validation_service_token + }.compact + Gitlab::HTTP.post( - validation_service_url, timeout: VALIDATION_REQUEST_TIMEOUT, - body: validation_service_payload(@pipeline, @command.yaml_processor_result.stages_attributes) + validation_service_url, timeout: validation_service_timeout, + headers: headers, + body: validation_service_payload.to_json ) end + def validation_service_timeout + timeout = Gitlab::CurrentSettings.external_pipeline_validation_service_timeout || ENV['EXTERNAL_VALIDATION_SERVICE_TIMEOUT'].to_i + return timeout if timeout > 0 + + DEFAULT_VALIDATION_REQUEST_TIMEOUT + end + def validation_service_url - ENV['EXTERNAL_VALIDATION_SERVICE_URL'] + Gitlab::CurrentSettings.external_pipeline_validation_service_url || ENV['EXTERNAL_VALIDATION_SERVICE_URL'] + end + + def validation_service_token + Gitlab::CurrentSettings.external_pipeline_validation_service_token || ENV['EXTERNAL_VALIDATION_SERVICE_TOKEN'] end - def validation_service_payload(pipeline, stages_attributes) + def validation_service_payload { project: { - id: pipeline.project.id, - path: pipeline.project.full_path + id: project.id, + path: project.full_path, + created_at: project.created_at&.iso8601 }, user: { - id: pipeline.user.id, - username: pipeline.user.username, - email: pipeline.user.email + id: current_user.id, + username: current_user.username, + email: current_user.email, + created_at: current_user.created_at&.iso8601 }, pipeline: { sha: pipeline.sha, ref: pipeline.ref, type: pipeline.source }, - builds: builds_validation_payload(stages_attributes) - }.to_json + builds: builds_validation_payload + } end - def builds_validation_payload(stages_attributes) - stages_attributes.map { |stage| stage[:builds] }.flatten + def builds_validation_payload + stages_attributes.flat_map { |stage| stage[:builds] } .map(&method(:build_validation_payload)) end @@ -97,9 +136,15 @@ module Gitlab ].flatten.compact } end + + def stages_attributes + command.yaml_processor_result.stages_attributes + end end end end end end end + +Gitlab::Ci::Pipeline::Chain::Validate::External.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Validate::External') diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index c77f4dcca5a..6cb6fd3920d 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -4,55 +4,57 @@ module Gitlab module Ci module Pipeline class Metrics - include Gitlab::Utils::StrongMemoize + def self.pipeline_creation_duration_histogram + name = :gitlab_ci_pipeline_creation_duration_seconds + comment = 'Pipeline creation duration' + labels = {} + buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] - def pipeline_creation_duration_histogram - strong_memoize(:pipeline_creation_duration_histogram) do - name = :gitlab_ci_pipeline_creation_duration_seconds - comment = 'Pipeline creation duration' - labels = {} - buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + + def self.pipeline_size_histogram + name = :gitlab_ci_pipeline_size_builds + comment = 'Pipeline size' + labels = { source: nil } + buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + + def self.pipeline_processing_events_counter + name = :gitlab_ci_pipeline_processing_events_total + comment = 'Total amount of pipeline processing events' - ::Gitlab::Metrics.histogram(name, comment, labels, buckets) - end + Gitlab::Metrics.counter(name, comment) end - def pipeline_size_histogram - strong_memoize(:pipeline_size_histogram) do - name = :gitlab_ci_pipeline_size_builds - comment = 'Pipeline size' - labels = { source: nil } - buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000] + def self.pipelines_created_counter + name = :pipelines_created_total + comment = 'Counter of pipelines created' - ::Gitlab::Metrics.histogram(name, comment, labels, buckets) - end + Gitlab::Metrics.counter(name, comment) end - def pipeline_processing_events_counter - strong_memoize(:pipeline_processing_events_counter) do - name = :gitlab_ci_pipeline_processing_events_total - comment = 'Total amount of pipeline processing events' + def self.legacy_update_jobs_counter + name = :ci_legacy_update_jobs_as_retried_total + comment = 'Counter of occurrences when jobs were not being set as retried before update_retried' - Gitlab::Metrics.counter(name, comment) - end + Gitlab::Metrics.counter(name, comment) end - def pipelines_created_counter - strong_memoize(:pipelines_created_count) do - name = :pipelines_created_total - comment = 'Counter of pipelines created' + def self.pipeline_failure_reason_counter + name = :gitlab_ci_pipeline_failure_reasons + comment = 'Counter of pipeline failure reasons' - Gitlab::Metrics.counter(name, comment) - end + Gitlab::Metrics.counter(name, comment) end - def legacy_update_jobs_counter - strong_memoize(:legacy_update_jobs_counter) do - name = :ci_legacy_update_jobs_as_retried_total - comment = 'Counter of occurrences when jobs were not being set as retried before update_retried' + def self.job_failure_reason_counter + name = :gitlab_ci_job_failure_reasons + comment = 'Counter of job failure reasons' - Gitlab::Metrics.counter(name, comment) - end + Gitlab::Metrics.counter(name, comment) end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 11b01822e4b..39dee7750d6 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,12 +11,15 @@ module Gitlab delegate :dig, to: :@seed_attributes - def initialize(pipeline, attributes, previous_stages) - @pipeline = pipeline + def initialize(context, attributes, previous_stages) + @context = context + @pipeline = context.pipeline @seed_attributes = attributes @previous_stages = previous_stages @needs_attributes = dig(:needs_attributes) @resource_group_key = attributes.delete(:resource_group_key) + @job_variables = @seed_attributes.delete(:job_variables) + @root_variables_inheritance = @seed_attributes.delete(:root_variables_inheritance) { true } @using_rules = attributes.key?(:rules) @using_only = attributes.key?(:only) @@ -29,7 +32,9 @@ module Gitlab @rules = Gitlab::Ci::Build::Rules .new(attributes.delete(:rules), default_when: 'on_success') @cache = Gitlab::Ci::Build::Cache - .new(attributes.delete(:cache), pipeline) + .new(attributes.delete(:cache), @pipeline) + + recalculate_yaml_variables! end def name @@ -206,6 +211,14 @@ module Gitlab { options: { allow_failure_criteria: nil } } end + + def recalculate_yaml_variables! + return unless ::Feature.enabled?(:ci_workflow_rules_variables, @pipeline.project, default_enabled: :yaml) + + @seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( + from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance + ) + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/context.rb b/lib/gitlab/ci/pipeline/seed/context.rb new file mode 100644 index 00000000000..6194a78f682 --- /dev/null +++ b/lib/gitlab/ci/pipeline/seed/context.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Seed + class Context + attr_reader :pipeline, :root_variables + + def initialize(pipeline, root_variables: []) + @pipeline = pipeline + @root_variables = root_variables + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/pipeline.rb b/lib/gitlab/ci/pipeline/seed/pipeline.rb index da9d853cf68..e1a15fb8d5b 100644 --- a/lib/gitlab/ci/pipeline/seed/pipeline.rb +++ b/lib/gitlab/ci/pipeline/seed/pipeline.rb @@ -7,8 +7,8 @@ module Gitlab class Pipeline include Gitlab::Utils::StrongMemoize - def initialize(pipeline, stages_attributes) - @pipeline = pipeline + def initialize(context, stages_attributes) + @context = context @stages_attributes = stages_attributes end @@ -37,7 +37,7 @@ module Gitlab def stage_seeds strong_memoize(:stage_seeds) do seeds = @stages_attributes.inject([]) do |previous_stages, attributes| - seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@pipeline, attributes, previous_stages) + seed = Gitlab::Ci::Pipeline::Seed::Stage.new(@context, attributes, previous_stages) previous_stages + [seed] end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index b600df2f656..c988ea10e41 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -10,13 +10,14 @@ module Gitlab delegate :size, to: :seeds delegate :dig, to: :seeds - def initialize(pipeline, attributes, previous_stages) - @pipeline = pipeline + def initialize(context, attributes, previous_stages) + @context = context + @pipeline = context.pipeline @attributes = attributes @previous_stages = previous_stages @builds = attributes.fetch(:builds).map do |attributes| - Seed::Build.new(@pipeline, attributes, previous_stages) + Seed::Build.new(context, attributes, previous_stages) end end diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb index 5398c19e536..7ecb9a1db16 100644 --- a/lib/gitlab/ci/queue/metrics.rb +++ b/lib/gitlab/ci/queue/metrics.rb @@ -9,12 +9,12 @@ module Gitlab QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze - QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze - QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze + QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 7500, 10000, 15000, 20000].freeze + QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze METRICS_SHARD_TAG_PREFIX = 'metrics_shard::' DEFAULT_METRICS_SHARD = 'default' - JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5.freeze + JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET = 5 OPERATION_COUNTERS = [ :build_can_pick, @@ -94,13 +94,13 @@ module Gitlab self.class.queue_depth_total.observe({ queue: queue }, size.to_f) end - def observe_queue_size(size_proc) + def observe_queue_size(size_proc, runner_type) return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) - self.class.queue_size_total.observe({}, size_proc.call.to_f) + self.class.queue_size_total.observe({ runner_type: runner_type }, size_proc.call.to_f) end - def observe_queue_time + def observe_queue_time(metric, runner_type) start_time = ::Gitlab::Metrics::System.monotonic_time result = yield @@ -108,7 +108,15 @@ module Gitlab return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false) seconds = ::Gitlab::Metrics::System.monotonic_time - start_time - self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f) + + case metric + when :process + self.class.queue_iteration_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f) + when :retrieve + self.class.queue_retrieval_duration_seconds.observe({ runner_type: runner_type }, seconds.to_f) + else + raise ArgumentError unless Rails.env.production? + end result end @@ -187,7 +195,18 @@ module Gitlab strong_memoize(:queue_iteration_duration_seconds) do name = :gitlab_ci_queue_iteration_duration_seconds comment = 'Time it takes to find a build in CI/CD queue' - buckets = QUEUE_ITERATION_DURATION_SECONDS_BUCKETS + buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS + labels = {} + + Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def self.queue_retrieval_duration_seconds + strong_memoize(:queue_retrieval_duration_seconds) do + name = :gitlab_ci_queue_retrieval_duration_seconds + comment = 'Time it takes to execute a SQL query to retrieve builds queue' + buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS labels = {} Gitlab::Metrics.histogram(name, comment, labels, buckets) diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb index 060a1e2399b..27c41c384b8 100644 --- a/lib/gitlab/ci/reports/codequality_reports.rb +++ b/lib/gitlab/ci/reports/codequality_reports.rb @@ -6,6 +6,7 @@ module Gitlab class CodequalityReports attr_reader :degradations, :error_message + SEVERITY_PRIORITIES = %w(blocker critical major minor info).map.with_index.to_h.freeze # { "blocker" => 0, "critical" => 1 ... } CODECLIMATE_SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'codeclimate.json').to_s def initialize @@ -29,12 +30,17 @@ module Gitlab @degradations.values end + def sort_degradations! + @degradations = @degradations.sort_by do |_fingerprint, degradation| + SEVERITY_PRIORITIES[degradation.dig(:severity)] + end.to_h + end + private def valid_degradation?(degradation) - JSON::Validator.validate!(CODECLIMATE_SCHEMA_PATH, degradation) - rescue JSON::Schema::ValidationError => e - set_error_message("Invalid degradation format: #{e.message}") + JSONSchemer.schema(Pathname.new(CODECLIMATE_SCHEMA_PATH)).valid?(degradation) + rescue StandardError => _ false end end diff --git a/lib/gitlab/ci/reports/codequality_reports_comparer.rb b/lib/gitlab/ci/reports/codequality_reports_comparer.rb index 10748b8ca02..e34d9675c10 100644 --- a/lib/gitlab/ci/reports/codequality_reports_comparer.rb +++ b/lib/gitlab/ci/reports/codequality_reports_comparer.rb @@ -7,6 +7,11 @@ module Gitlab def initialize(base_report, head_report) @base_report = base_report @head_report = head_report + + unless not_found? + @base_report.sort_degradations! + @head_report.sort_degradations! + end end def success? diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb index c024e794ad5..37d0da38065 100644 --- a/lib/gitlab/ci/reports/test_failure_history.rb +++ b/lib/gitlab/ci/reports/test_failure_history.rb @@ -6,32 +6,32 @@ module Gitlab class TestFailureHistory include Gitlab::Utils::StrongMemoize - def initialize(failed_test_cases, project) - @failed_test_cases = build_map(failed_test_cases) + def initialize(failed_junit_tests, project) + @failed_junit_tests = build_map(failed_junit_tests) @project = project end def load! recent_failures_count.each do |key_hash, count| - failed_test_cases[key_hash].set_recent_failures(count, project.default_branch_or_master) + failed_junit_tests[key_hash].set_recent_failures(count, project.default_branch_or_master) end end private - attr_reader :report, :project, :failed_test_cases + attr_reader :report, :project, :failed_junit_tests def recent_failures_count - ::Ci::TestCaseFailure.recent_failures_count( + ::Ci::UnitTestFailure.recent_failures_count( project: project, - test_case_keys: failed_test_cases.keys + unit_test_keys: failed_junit_tests.keys ) end - def build_map(test_cases) + def build_map(junit_tests) {}.tap do |hash| - test_cases.each do |test_case| - hash[test_case.key] = test_case + junit_tests.each do |test| + hash[test.key] = test end end end diff --git a/lib/gitlab/ci/runner_instructions.rb b/lib/gitlab/ci/runner_instructions.rb index dd0bfa768a8..365864d3317 100644 --- a/lib/gitlab/ci/runner_instructions.rb +++ b/lib/gitlab/ci/runner_instructions.rb @@ -51,10 +51,7 @@ module Gitlab attr_reader :errors - def initialize(current_user:, group: nil, project: nil, os:, arch:) - @current_user = current_user - @group = group - @project = project + def initialize(os:, arch:) @os = os @arch = arch @errors = [] @@ -77,7 +74,7 @@ module Gitlab server_url = Gitlab::Routing.url_helpers.root_url(only_path: false) runner_executable = environment[:runner_executable] - "#{runner_executable} register --url #{server_url} --registration-token #{registration_token}" + "#{runner_executable} register --url #{server_url} --registration-token $REGISTRATION_TOKEN" end end @@ -108,30 +105,6 @@ module Gitlab def get_file(path) File.read(Rails.root.join(path).to_s) end - - def registration_token - project_token || group_token || instance_token - end - - def project_token - return unless @project - raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_pipeline, @project) - - @project.runners_token - end - - def group_token - return unless @group - raise Gitlab::Access::AccessDeniedError unless can?(@current_user, :admin_group, @group) - - @group.runners_token - end - - def instance_token - raise Gitlab::Access::AccessDeniedError unless @current_user&.admin? - - Gitlab::CurrentSettings.runners_registration_token - end end end end diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index f6562737838..787dee3b267 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -26,7 +26,9 @@ module Gitlab bridge_pipeline_is_child_pipeline: 'creation of child pipeline not allowed from another child pipeline', downstream_pipeline_creation_failed: 'downstream pipeline can not be created', secrets_provider_not_found: 'secrets provider can not be found', - reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines' + reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines', + project_deleted: 'pipeline project was deleted', + user_blocked: 'pipeline user was blocked' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml index 5ebbbf15682..2ff36bcc657 100644 --- a/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Android-Fastlane.gitlab-ci.yml @@ -113,9 +113,10 @@ promoteBeta: promoteProduction: extends: .promote_job stage: production - # We only allow production promotion on `master` because - # it has its own production scoped secret variables + # We only allow production promotion on the default branch because + # it has its own production scoped secret variables. only: - - master + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH script: - bundle exec fastlane promote_beta_to_production diff --git a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml index 15cdbf63cb1..d0c63ab6edf 100644 --- a/lib/gitlab/ci/templates/Docker.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Docker.gitlab-ci.yml @@ -1,27 +1,31 @@ -docker-build-master: - # Official docker image. - image: docker:latest - stage: build - services: - - docker:dind - before_script: - - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY - script: - - docker build --pull -t "$CI_REGISTRY_IMAGE" . - - docker push "$CI_REGISTRY_IMAGE" - only: - - master - +# Build a Docker image with CI/CD and push to the GitLab registry. +# Docker-in-Docker documentation: https://docs.gitlab.com/ee/ci/docker/using_docker_build.html +# +# This template uses one generic job with conditional builds +# for the default branch and all other (MR) branches. docker-build: - # Official docker image. + # Use the official docker image. image: docker:latest stage: build services: - docker:dind before_script: - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY + # Default branch leaves tag empty (= latest tag) + # All other branches are tagged with the escaped branch name (commit ref slug) script: - - docker build --pull -t "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" . - - docker push "$CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG" - except: - - master + - | + if [[ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]]; then + tag="" + echo "Running on default branch '$CI_DEFAULT_BRANCH': tag = 'latest'" + else + tag=":$CI_COMMIT_REF_SLUG" + echo "Running on branch '$CI_COMMIT_BRANCH': tag = $tag" + fi + - docker build --pull -t "$CI_REGISTRY_IMAGE${tag}" . + - docker push "$CI_REGISTRY_IMAGE${tag}" + # Run this job in a branch where a Dockerfile exists + rules: + - if: $CI_COMMIT_BRANCH + exists: + - Dockerfile diff --git a/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml new file mode 100644 index 00000000000..90812083917 --- /dev/null +++ b/lib/gitlab/ci/templates/Hello-World.gitlab-ci.yml @@ -0,0 +1,9 @@ +# This file is a template demonstrating the `script` keyword. +# Learn more about this keyword here: https://docs.gitlab.com/ee/ci/yaml/README.html#script + +# After committing this template, visit CI/CD > Jobs to see the script output. + +job: + script: + # provide a shell script as argument for this keyword. + - echo "Hello World" diff --git a/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml new file mode 100644 index 00000000000..c7fb1321055 --- /dev/null +++ b/lib/gitlab/ci/templates/Indeni.Cloudrail.gitlab-ci-.yml @@ -0,0 +1,91 @@ +# This template is provided and maintained by Indeni, an official Technology Partner with GitLab. +# See https://about.gitlab.com/partners/technology-partners/#security for more information. + +# For more information about Indeni Cloudrail: https://indeni.com/cloudrail/ +# +# This file shows an example of using Indeni Cloudrail with GitLab CI/CD. +# It is not designed to be included in an existing CI/CD configuration with the "include:" keyword. +# Documentation about this integration: https://indeni.com/doc-indeni-cloudrail/integrate-with-ci-cd/gitlab-instructions +# +# For an example of this used in a GitLab repository, see: https://gitlab.com/indeni/cloudrail-demo/-/blob/master/.gitlab-ci.yml + +# The sast-report output complies with GitLab's format. This report displays Cloudrail's +# results in the Security tab in the pipeline view, if you have that feature enabled +# (GitLab Ultimate only). Otherwise, Cloudrail generates a JUnit report, which displays +# in the "Test summary" in merge requests. + +# Note that Cloudrail's input is the Terraform plan. That is why we've included in this +# template an example of doing that. You are welcome to replace it with your own way +# of generating a Terraform plan. + +# Before you can use this template, get a Cloudrail API key from the Cloudrail web +# user interface. Save it as a CI/CD variable named CLOUDRAIL_API_KEY in your project +# settings. + +variables: + TEST_ROOT: ${CI_PROJECT_DIR}/my_folder_with_terraform_content + +default: + before_script: + - cd ${CI_PROJECT_DIR}/my_folder_with_terraform_content + +stages: + - init_and_plan + - cloudrail + +init_and_plan: + stage: init_and_plan + image: registry.gitlab.com/gitlab-org/terraform-images/releases/0.13 + rules: + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.tf' + script: + - terraform init + - terraform plan -out=plan.out + artifacts: + name: "$CI_COMMIT_BRANCH-terraform_plan" + paths: + - ./**/plan.out + - ./**/.terraform + +cloudrail_scan: + stage: cloudrail + image: indeni/cloudrail-cli:1.2.44 + rules: + - if: $SAST_DISABLED + when: never + - if: $CI_COMMIT_BRANCH + exists: + - '**/*.tf' + script: + - | + if [[ "${GITLAB_FEATURES}" == *"security_dashboard"* ]]; then + echo "You are licensed for GitLab Security Dashboards. Your scan results will display in the Security Dashboard." + cloudrail run --tf-plan plan.out \ + --directory . \ + --api-key ${CLOUDRAIL_API_KEY} \ + --origin ci \ + --build-link "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID" \ + --execution-source-identifier "$CI_COMMIT_BRANCH - $CI_JOB_ID" \ + --output-format json-gitlab-sast \ + --output-file ${CI_PROJECT_DIR}/cloudrail-sast-report.json \ + --auto-approve + else + echo "Your scan results will display in the GitLab Test results visualization panel." + cloudrail run --tf-plan plan.out \ + --directory . \ + --api-key ${CLOUDRAIL_API_KEY} \ + --origin ci \ + --build-link "$CI_PROJECT_URL/-/jobs/$CI_JOB_ID" \ + --execution-source-identifier "$CI_COMMIT_BRANCH - $CI_JOB_ID" \ + --output-format junit \ + --output-file ${CI_PROJECT_DIR}/cloudrail-junit-report.xml \ + --auto-approve + fi + artifacts: + reports: + sast: cloudrail-sast-report.json + junit: cloudrail-junit-report.xml diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index 5edb26a0b56..01907ef9e2e 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -20,15 +20,48 @@ performance: fi - export CI_ENVIRONMENT_URL=$(cat environment_url.txt) - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget - wget -O gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | if [ -f .gitlab-urls.txt ] then sed -i -e 's@^@'"$CI_ENVIRONMENT_URL"'@' .gitlab-urls.txt - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results .gitlab-urls.txt $SITESPEED_OPTIONS else - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results "$CI_ENVIRONMENT_URL" $SITESPEED_OPTIONS fi - mv sitespeed-results/data/performance.json browser-performance.json artifacts: diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 1c25d9d583b..196d42f3e3a 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,10 +1,10 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.4.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.6.0" variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.12-dind + - docker:20.10.6-dind script: - | if [[ -z "$CI_COMMIT_TAG" ]]; then diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index fd6c51ea350..b29342216fc 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -36,6 +36,7 @@ code_quality: REPORT_STDOUT \ REPORT_FORMAT \ ENGINE_MEMORY_LIMIT_BYTES \ + CODECLIMATE_PREFIX \ ) \ --volume "$PWD":/code \ --volume /var/run/docker.sock:/var/run/docker.sock \ diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index 654a03ced5f..bf42cd52605 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -12,7 +12,7 @@ stages: variables: FUZZAPI_PROFILE: Quick - FUZZAPI_VERSION: latest + FUZZAPI_VERSION: "1.6" FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml FUZZAPI_TIMEOUT: 30 FUZZAPI_REPORT: gl-api-fuzzing-report.json @@ -45,7 +45,7 @@ apifuzzer_fuzz: entrypoint: ["/bin/bash", "-l", "-c"] variables: FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://localhost:80 + FUZZAPI_API: http://localhost:5000 FUZZAPI_NEW_REPORT: 1 FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log TZ: America/Los_Angeles @@ -107,7 +107,7 @@ apifuzzer_fuzz_dnd: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" FUZZAPI_PROJECT: $CI_PROJECT_PATH - FUZZAPI_API: http://apifuzzer:80 + FUZZAPI_API: http://apifuzzer:5000 allow_failure: true rules: - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null @@ -142,6 +142,7 @@ apifuzzer_fuzz_dnd: -e TZ=America/Los_Angeles \ -e GITLAB_FEATURES \ -p 80:80 \ + -p 5000:5000 \ -p 8000:8000 \ -p 514:514 \ --restart=no \ @@ -168,7 +169,7 @@ apifuzzer_fuzz_dnd: docker run \ --name worker \ --network $FUZZAPI_D_NETWORK \ - -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_API=http://apifuzzer:5000 \ -e FUZZAPI_PROJECT \ -e FUZZAPI_PROFILE \ -e FUZZAPI_CONFIG \ @@ -211,7 +212,7 @@ apifuzzer_fuzz_dnd: --name worker \ --network $FUZZAPI_D_NETWORK \ -e TZ=America/Los_Angeles \ - -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_API=http://apifuzzer:5000 \ -e FUZZAPI_PROJECT \ -e FUZZAPI_PROFILE \ -e FUZZAPI_CONFIG \ @@ -237,6 +238,7 @@ apifuzzer_fuzz_dnd: -v $CI_PROJECT_DIR:/app \ -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ -p 81:80 \ + -p 5001:5000 \ -p 8001:8000 \ -p 515:514 \ --restart=no \ diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml new file mode 100644 index 00000000000..215029dc952 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -0,0 +1,270 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ + +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/api_fuzzing/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + FUZZAPI_PROFILE: Quick + FUZZAPI_VERSION: latest + FUZZAPI_CONFIG: .gitlab-api-fuzzing.yml + FUZZAPI_TIMEOUT: 30 + FUZZAPI_REPORT: gl-api-fuzzing-report.json + FUZZAPI_REPORT_ASSET_PATH: assets + # + FUZZAPI_D_NETWORK: testing-net + # + # Wait up to 5 minutes for API Fuzzer and target url to become + # available (non 500 response to HTTP(s)) + FUZZAPI_SERVICE_START_TIMEOUT: "300" + # + FUZZAPI_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${FUZZAPI_VERSION}-engine + # + +apifuzzer_fuzz_unlicensed: + stage: fuzz + allow_failure: true + rules: + - if: '$GITLAB_FEATURES !~ /\bapi_fuzzing\b/ && $API_FUZZING_DISABLED == null' + - when: never + script: + - | + echo "Error: Your GitLab project is not licensed for API Fuzzing." + - exit 1 + +apifuzzer_fuzz: + stage: fuzz + image: + name: $FUZZAPI_IMAGE + entrypoint: ["/bin/bash", "-l", "-c"] + variables: + FUZZAPI_PROJECT: $CI_PROJECT_PATH + FUZZAPI_API: http://localhost:80 + FUZZAPI_NEW_REPORT: 1 + FUZZAPI_LOG_SCANNER: gl-apifuzzing-api-scanner.log + TZ: America/Los_Angeles + allow_failure: true + rules: + - if: $FUZZAPI_D_TARGET_IMAGE + when: never + - if: $FUZZAPI_D_WORKER_IMAGE + when: never + - if: $API_FUZZING_DISABLED + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + script: + # + # Validate options + - | + if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ + echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ + echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ + exit 1; \ + fi + # + # Run user provided pre-script + - sh -c "$FUZZAPI_PRE_SCRIPT" + # + # Make sure asset path exists + - mkdir -p $FUZZAPI_REPORT_ASSET_PATH + # + # Start API Security background process + - dotnet /peach/Peach.Web.dll &> $FUZZAPI_LOG_SCANNER & + - APISEC_PID=$! + # + # Start scanning + - worker-entry + # + # Run user provided post-script + - sh -c "$FUZZAPI_POST_SCRIPT" + # + # Shutdown API Security + - kill $APISEC_PID + - wait $APISEC_PID + # + artifacts: + when: always + paths: + - $FUZZAPI_REPORT_ASSET_PATH + - $FUZZAPI_REPORT + - $FUZZAPI_LOG_SCANNER + reports: + api_fuzzing: $FUZZAPI_REPORT + +apifuzzer_fuzz_dnd: + stage: fuzz + image: docker:19.03.12 + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + FUZZAPI_PROJECT: $CI_PROJECT_PATH + FUZZAPI_API: http://apifuzzer:80 + allow_failure: true + rules: + - if: $FUZZAPI_D_TARGET_IMAGE == null && $FUZZAPI_D_WORKER_IMAGE == null + when: never + - if: $API_FUZZING_DISABLED + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bapi_fuzzing\b/ + services: + - docker:19.03.12-dind + script: + # + # + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + # + - docker network create --driver bridge $FUZZAPI_D_NETWORK + # + # Run user provided pre-script + - sh -c "$FUZZAPI_PRE_SCRIPT" + # + # Make sure asset path exists + - mkdir -p $FUZZAPI_REPORT_ASSET_PATH + # + # Start peach testing engine container + - | + docker run -d \ + --name apifuzzer \ + --network $FUZZAPI_D_NETWORK \ + -e Proxy:Port=8000 \ + -e TZ=America/Los_Angeles \ + -e GITLAB_FEATURES \ + -p 80:80 \ + -p 8000:8000 \ + -p 514:514 \ + --restart=no \ + $FUZZAPI_IMAGE \ + dotnet /peach/Peach.Web.dll + # + # Start target container + - | + if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then \ + docker run -d \ + --name target \ + --network $FUZZAPI_D_NETWORK \ + $FUZZAPI_D_TARGET_ENV \ + $FUZZAPI_D_TARGET_PORTS \ + $FUZZAPI_D_TARGET_VOLUME \ + --restart=no \ + $FUZZAPI_D_TARGET_IMAGE \ + ; fi + # + # Start worker container if provided + - | + if [ "$FUZZAPI_D_WORKER_IMAGE" != "" ]; then \ + echo "Starting worker image $FUZZAPI_D_WORKER_IMAGE"; \ + docker run \ + --name worker \ + --network $FUZZAPI_D_NETWORK \ + -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e FUZZAPI_REPORT_ASSET_PATH \ + -e FUZZAPI_NEW_REPORT=1 \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ + -e CI_PROJECT_URL \ + -e CI_JOB_ID \ + -e CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH} \ + $FUZZAPI_D_WORKER_ENV \ + $FUZZAPI_D_WORKER_PORTS \ + $FUZZAPI_D_WORKER_VOLUME \ + --restart=no \ + $FUZZAPI_D_WORKER_IMAGE \ + ; fi + # + # Start API Fuzzing provided worker if no other worker present + - | + if [ "$FUZZAPI_D_WORKER_IMAGE" == "" ]; then \ + if [ "$FUZZAPI_HAR$FUZZAPI_OPENAPI$FUZZAPI_POSTMAN_COLLECTION" == "" ]; then \ + echo "Error: One of FUZZAPI_HAR, FUZZAPI_OPENAPI, or FUZZAPI_POSTMAN_COLLECTION must be provided."; \ + echo "See https://docs.gitlab.com/ee/user/application_security/api_fuzzing/ for information on how to configure API Fuzzing."; \ + exit 1; \ + fi; \ + docker run \ + --name worker \ + --network $FUZZAPI_D_NETWORK \ + -e TZ=America/Los_Angeles \ + -e FUZZAPI_API=http://apifuzzer:80 \ + -e FUZZAPI_PROJECT \ + -e FUZZAPI_PROFILE \ + -e FUZZAPI_CONFIG \ + -e FUZZAPI_REPORT \ + -e FUZZAPI_REPORT_ASSET_PATH \ + -e FUZZAPI_NEW_REPORT=1 \ + -e FUZZAPI_HAR \ + -e FUZZAPI_OPENAPI \ + -e FUZZAPI_POSTMAN_COLLECTION \ + -e FUZZAPI_POSTMAN_COLLECTION_VARIABLES \ + -e FUZZAPI_TARGET_URL \ + -e FUZZAPI_OVERRIDES_FILE \ + -e FUZZAPI_OVERRIDES_ENV \ + -e FUZZAPI_OVERRIDES_CMD \ + -e FUZZAPI_OVERRIDES_INTERVAL \ + -e FUZZAPI_TIMEOUT \ + -e FUZZAPI_VERBOSE \ + -e FUZZAPI_SERVICE_START_TIMEOUT \ + -e FUZZAPI_HTTP_USERNAME \ + -e FUZZAPI_HTTP_PASSWORD \ + -e CI_PROJECT_URL \ + -e CI_JOB_ID \ + -v $CI_PROJECT_DIR:/app \ + -v `pwd`/$FUZZAPI_REPORT_ASSET_PATH:/app/$FUZZAPI_REPORT_ASSET_PATH:rw \ + -p 81:80 \ + -p 8001:8000 \ + -p 515:514 \ + --restart=no \ + $FUZZAPI_IMAGE \ + worker-entry \ + ; fi + # + # Propagate exit code from api fuzzing scanner (if any) + - if [[ $(docker inspect apifuzzer --format='{{.State.ExitCode}}') != "0" ]]; then echo "API Fuzzing scanner exited with an error. Logs are available as job artifacts."; exit 1; fi + # + # Run user provided post-script + - sh -c "$FUZZAPI_POST_SCRIPT" + # + after_script: + # + # Shutdown all containers + - echo "Stopping all containers" + - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker stop target; fi + - docker stop worker + - docker stop apifuzzer + # + # Save docker logs + - docker logs apifuzzer &> gl-api_fuzzing-logs.log + - if [ "$FUZZAPI_D_TARGET_IMAGE" != "" ]; then docker logs target &> gl-api_fuzzing-target-logs.log; fi + - docker logs worker &> gl-api_fuzzing-worker-logs.log + # + artifacts: + when: always + paths: + - ./gl-api_fuzzing*.log + - ./gl-api_fuzzing*.zip + - $FUZZAPI_REPORT_ASSET_PATH + - $FUZZAPI_REPORT + reports: + api_fuzzing: $FUZZAPI_REPORT + +# end diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 64001c2828a..c628e30b2c7 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -6,14 +6,10 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" CS_MAJOR_VERSION: 3 -container_scanning: +.cs_common: stage: test image: "$CS_ANALYZER_IMAGE" variables: - # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image - # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes - CLAIR_DB_IMAGE_TAG: "latest" - CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" # Override the GIT_STRATEGY variable in your `.gitlab-ci.yml` file and set it to `fetch` if you want to provide a `clair-whitelist.yml` # file. See https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template # for details @@ -21,19 +17,44 @@ container_scanning: # CS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to # override the analyzer image with a custom value. This may be subject to change or # breakage across GitLab releases. - CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/klar:$CS_MAJOR_VERSION + CS_ANALYZER_IMAGE: $SECURE_ANALYZERS_PREFIX/$CS_PROJECT:$CS_MAJOR_VERSION allow_failure: true + artifacts: + reports: + container_scanning: gl-container-scanning-report.json + dependencies: [] + +container_scanning: + extends: .cs_common + variables: + # By default, use the latest clair vulnerabilities database, however, allow it to be overridden here with a specific image + # to enable container scanning to run offline, or to provide a consistent list of vulnerabilities for integration testing purposes + CLAIR_DB_IMAGE_TAG: "latest" + CLAIR_DB_IMAGE: "$SECURE_ANALYZERS_PREFIX/clair-vulnerabilities-db:$CLAIR_DB_IMAGE_TAG" + CS_PROJECT: 'klar' services: - name: $CLAIR_DB_IMAGE alias: clair-vulnerabilities-db script: - /analyzer run + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && + $CS_MAJOR_VERSION =~ /^[0-3]$/ + +container_scanning_new: + extends: .cs_common + variables: + CS_PROJECT: 'container-scanning' + script: + - gtcs scan artifacts: - reports: - container_scanning: gl-container-scanning-report.json - dependencies: [] + paths: [gl-container-scanning-report.json] rules: - if: $CONTAINER_SCANNING_DISABLED when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ + $GITLAB_FEATURES =~ /\bcontainer_scanning\b/ && + $CS_MAJOR_VERSION !~ /^[0-3]$/ diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index fc1acd09714..533f8bb25f8 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -1,3 +1,16 @@ +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: DAST.latest.gitlab-ci.yml +# +# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST: +# +# stages: +# - build +# - test +# - deploy +# - dast + # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ # Configure the scanning tool through the environment variables. @@ -9,6 +22,19 @@ variables: # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + # + DAST_API_PROFILE: Full + DAST_API_VERSION: latest + DAST_API_CONFIG: .gitlab-dast-api.yml + DAST_API_TIMEOUT: 30 + DAST_API_REPORT: gl-dast-api-report.json + DAST_API_REPORT_ASSET_PATH: assets + # + # Wait up to 5 minutes for API Security and target url to become + # available (non 500 response to HTTP(s)) + DAST_API_SERVICE_START_TIMEOUT: "300" + # + DAST_API_IMAGE: registry.gitlab.com/gitlab-org/security-products/analyzers/api-fuzzing:${DAST_API_VERSION}-engine dast: stage: dast @@ -25,6 +51,11 @@ dast: reports: dast: gl-dast-report.json rules: + - if: $DAST_API_BETA && ( $DAST_API_SPECIFICATION || + $DAST_API_OPENAPI || + $DAST_API_POSTMAN_COLLECTION || + $DAST_API_HAR ) + when: never - if: $DAST_DISABLED when: never - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && @@ -40,4 +71,72 @@ dast: - if: $CI_COMMIT_BRANCH && $DAST_WEBSITE - if: $CI_COMMIT_BRANCH && + $DAST_API_BETA == null && $DAST_API_SPECIFICATION + +dast_api: + stage: dast + image: + name: $DAST_API_IMAGE + entrypoint: ["/bin/bash", "-l", "-c"] + variables: + API_SECURITY_MODE: DAST + DAST_API_NEW_REPORT: 1 + DAST_API_PROJECT: $CI_PROJECT_PATH + DAST_API_API: http://127.0.0.1:5000 + DAST_API_LOG_SCANNER: gl-dast-api-scanner.log + TZ: America/Los_Angeles + allow_failure: true + rules: + - if: $DAST_API_BETA == null + when: never + - if: $DAST_DISABLED + when: never + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && + $REVIEW_DISABLED && + $DAST_API_SPECIFICATION == null && + $DAST_API_OPENAPI == null && + $DAST_API_POSTMAN_COLLECTION == null && + $DAST_API_HAR == null + when: never + - if: $DAST_API_SPECIFICATION == null && + $DAST_API_OPENAPI == null && + $DAST_API_POSTMAN_COLLECTION == null && + $DAST_API_HAR == null + when: never + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdast\b/ + script: + # + # Run user provided pre-script + - sh -c "$DAST_API_PRE_SCRIPT" + # + # Make sure asset path exists + - mkdir -p $DAST_API_REPORT_ASSET_PATH + # + # Start API Security background process + - dotnet /peach/Peach.Web.dll &> $DAST_API_LOG_SCANNER & + - APISEC_PID=$! + # + # Start scanning + - worker-entry + # + # Run user provided post-script + - sh -c "$DAST_API_POST_SCRIPT" + # + # Shutdown API Security + - kill $APISEC_PID + - wait $APISEC_PID + # + artifacts: + when: always + paths: + - $DAST_API_REPORT_ASSET_PATH + - $DAST_API_REPORT + - $DAST_API_LOG_SCANNER + - gl-*.log + reports: + dast: $DAST_API_REPORT diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 9693a4fbca2..3ebccfbba4a 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -160,7 +160,7 @@ mobsf-android-sast: services: # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.3.3 + - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" @@ -186,7 +186,7 @@ mobsf-ios-sast: services: # this version must match with analyzer version mentioned in: https://gitlab.com/gitlab-org/security-products/analyzers/mobsf/-/blob/master/Dockerfile # Unfortunately, we need to keep track of mobsf version in 2 different places for now. - - name: opensecurity/mobile-security-framework-mobsf:v3.3.3 + - name: opensecurity/mobile-security-framework-mobsf:v3.4.0 alias: mobsf image: name: "$SAST_ANALYZER_IMAGE" @@ -303,6 +303,10 @@ semgrep-sast: $SAST_EXPERIMENTAL_FEATURES == 'true' exists: - '**/*.py' + - '**/*.js' + - '**/*.jsx' + - '**/*.ts' + - '**/*.tsx' sobelow-sast: extends: .sast-analyzer @@ -348,3 +352,4 @@ spotbugs-sast: - '**/*.groovy' - '**/*.java' - '**/*.scala' + - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index e591e3cc1e2..404d4a4c6db 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -18,9 +18,32 @@ performance: - docker:stable-dind script: - mkdir gitlab-exporter + # Busybox wget does not support proxied HTTPS, get the real thing. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/287611. + - (env | grep -i _proxy= 2>&1 >/dev/null) && apk --no-cache add wget - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/1.1.0/index.js - mkdir sitespeed-results - - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + - | + function propagate_env_vars() { + CURRENT_ENV=$(printenv) + + for VAR_NAME; do + echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME " + done + } + - | + docker run \ + $(propagate_env_vars \ + auto_proxy \ + https_proxy \ + http_proxy \ + no_proxy \ + AUTO_PROXY \ + HTTPS_PROXY \ + HTTP_PROXY \ + NO_PROXY \ + ) \ + --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS - mv sitespeed-results/data/performance.json browser-performance.json artifacts: paths: diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 3258d965c93..c25c4339c35 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -11,7 +11,7 @@ module Gitlab LOCK_SLEEP = 0.001.seconds WATCH_FLAG_TTL = 10.seconds - UPDATE_FREQUENCY_DEFAULT = 30.seconds + UPDATE_FREQUENCY_DEFAULT = 60.seconds UPDATE_FREQUENCY_WHEN_BEING_WATCHED = 3.seconds ArchiveError = Class.new(StandardError) @@ -93,6 +93,10 @@ module Gitlab end end + def erase_trace_chunks! + job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace + end + def erase! ## # Erase the archived trace @@ -100,7 +104,7 @@ module Gitlab ## # Erase the live trace - job.trace_chunks.fast_destroy_all # Destroy chunks of a live trace + erase_trace_chunks! FileUtils.rm_f(current_path) if current_path # Remove a trace file of a live trace job.erase_old_trace! if job.has_old_trace? # Remove a trace in database of a live trace ensure @@ -114,7 +118,11 @@ module Gitlab end def update_interval - being_watched? ? UPDATE_FREQUENCY_WHEN_BEING_WATCHED : UPDATE_FREQUENCY_DEFAULT + if being_watched? + UPDATE_FREQUENCY_WHEN_BEING_WATCHED + else + UPDATE_FREQUENCY_DEFAULT + end end def being_watched! @@ -176,9 +184,14 @@ module Gitlab end def unsafe_archive! - raise AlreadyArchivedError, 'Could not archive again' if trace_artifact raise ArchiveError, 'Job is not finished yet' unless job.complete? + if trace_artifact + unsafe_trace_cleanup! if Feature.enabled?(:erase_traces_from_already_archived_jobs_when_archiving_again, job.project, default_enabled: :yaml) + + raise AlreadyArchivedError, 'Could not archive again' + end + if job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| archive_stream!(stream) @@ -197,6 +210,18 @@ module Gitlab end end + def unsafe_trace_cleanup! + return unless trace_artifact + + if trace_artifact.archived_trace_exists? + # An archive already exists, so make sure to remove the trace chunks + erase_trace_chunks! + else + # An archive already exists, but its associated file does not, so remove it + trace_artifact.destroy! + end + end + def in_write_lock(&blk) lock_key = "trace:write:lock:#{job.id}" in_lock(lock_key, ttl: LOCK_TTL, retries: LOCK_RETRIES, sleep_sec: LOCK_SLEEP, &blk) diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb index e2a54f90ecb..3a62f01e2e3 100644 --- a/lib/gitlab/ci/variables/helpers.rb +++ b/lib/gitlab/ci/variables/helpers.rb @@ -23,7 +23,21 @@ module Gitlab def transform_from_yaml_variables(vars) return vars.stringify_keys if vars.is_a?(Hash) - vars.to_a.map { |var| [var[:key].to_s, var[:value]] }.to_h + vars.to_a.to_h { |var| [var[:key].to_s, var[:value]] } + end + + def inherit_yaml_variables(from:, to:, inheritance:) + merge_variables(apply_inheritance(from, inheritance), to) + end + + private + + def apply_inheritance(variables, inheritance) + case inheritance + when true then variables + when false then {} + when Array then variables.select { |var| inheritance.include?(var[:key]) } + end end end end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 3459b69bebc..f96a6629849 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -38,11 +38,12 @@ module Gitlab .map { |job| build_attributes(job[:name]) } end - def workflow_attributes - { - rules: hash_config.dig(:workflow, :rules), - yaml_variables: transform_to_yaml_variables(variables) - } + def workflow_rules + @workflow_rules ||= hash_config.dig(:workflow, :rules) + end + + def root_variables + @root_variables ||= transform_to_yaml_variables(variables) end def jobs @@ -68,7 +69,9 @@ module Gitlab when: job[:when] || 'on_success', environment: job[:environment_name], coverage_regex: job[:coverage], - yaml_variables: transform_to_yaml_variables(job[:variables]), + yaml_variables: transform_to_yaml_variables(job[:variables]), # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + job_variables: transform_to_yaml_variables(job[:job_variables]), + root_variables_inheritance: job[:root_variables_inheritance], needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], only: job[:only], @@ -101,7 +104,7 @@ module Gitlab end def merged_yaml - @ci_config&.to_hash&.to_yaml + @ci_config&.to_hash&.deep_stringify_keys&.to_yaml end def variables_with_data diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb index ac0071cdc53..fdff8fb32d3 100644 --- a/lib/gitlab/composer/version_index.rb +++ b/lib/gitlab/composer/version_index.rb @@ -28,20 +28,34 @@ module Gitlab def package_metadata(package) json = package.composer_metadatum.composer_json - json.merge('dist' => package_dist(package), 'uid' => package.id, 'version' => package.version) + json.merge( + 'dist' => package_dist(package), + 'source' => package_source(package), + 'uid' => package.id, + 'version' => package.version + ) end def package_dist(package) - sha = package.composer_metadatum.target_sha archive_api_path = api_v4_projects_packages_composer_archives_package_name_path({ id: package.project_id, package_name: package.name, format: '.zip' }, true) { 'type' => 'zip', - 'url' => expose_url(archive_api_path) + "?sha=#{sha}", - 'reference' => sha, + 'url' => expose_url(archive_api_path) + "?sha=#{package.composer_target_sha}", + 'reference' => package.composer_target_sha, 'shasum' => '' } end + + def package_source(package) + git_url = package.project.http_url_to_repo + + { + 'type' => 'git', + 'url' => git_url, + 'reference' => package.composer_target_sha + } + end end end end diff --git a/lib/gitlab/conan_token.rb b/lib/gitlab/conan_token.rb index d03997b4158..c3d90aa78fb 100644 --- a/lib/gitlab/conan_token.rb +++ b/lib/gitlab/conan_token.rb @@ -7,7 +7,7 @@ module Gitlab class ConanToken - HMAC_KEY = 'gitlab-conan-packages'.freeze + HMAC_KEY = 'gitlab-conan-packages' attr_reader :access_token_id, :user_id diff --git a/lib/gitlab/contributor.rb b/lib/gitlab/contributor.rb index d74d5a86aa0..c1c270bc9e6 100644 --- a/lib/gitlab/contributor.rb +++ b/lib/gitlab/contributor.rb @@ -5,7 +5,9 @@ module Gitlab attr_accessor :email, :name, :commits, :additions, :deletions def initialize - @commits, @additions, @deletions = 0, 0, 0 + @commits = 0 + @additions = 0 + @deletions = 0 end end end diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb index 4428354642d..c113cebd72f 100644 --- a/lib/gitlab/crypto_helper.rb +++ b/lib/gitlab/crypto_helper.rb @@ -16,34 +16,16 @@ module Gitlab ::Digest::SHA256.base64digest("#{value}#{salt}") end - def aes256_gcm_encrypt(value, nonce: nil) - aes256_gcm_encrypt_using_static_nonce(value) + def aes256_gcm_encrypt(value, nonce: AES256_GCM_IV_STATIC) + encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value, iv: nonce)) + Base64.strict_encode64(encrypted_token) end - def aes256_gcm_decrypt(value) + def aes256_gcm_decrypt(value, nonce: AES256_GCM_IV_STATIC) return unless value - nonce = Feature.enabled?(:dynamic_nonce_creation) ? dynamic_nonce(value) : AES256_GCM_IV_STATIC encrypted_token = Base64.decode64(value) - decrypted_token = Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token, iv: nonce)) - decrypted_token - end - - def dynamic_nonce(value) - TokenWithIv.find_nonce_by_hashed_token(value) || AES256_GCM_IV_STATIC - end - - def aes256_gcm_encrypt_using_static_nonce(value) - create_encrypted_token(value, AES256_GCM_IV_STATIC) - end - - def read_only? - Gitlab::Database.read_only? - end - - def create_encrypted_token(value, iv) - encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value, iv: iv)) - Base64.strict_encode64(encrypted_token) + Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token, iv: nonce)) end end end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index c4af5e6608e..0e4fc8efa95 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -12,7 +12,7 @@ module Gitlab author_url = build_author_url(build.commit, commit) - data = { + { object_kind: 'build', ref: build.ref, @@ -26,6 +26,7 @@ module Gitlab build_name: build.name, build_stage: build.stage, build_status: build.status, + build_created_at: build.created_at, build_started_at: build.started_at, build_finished_at: build.finished_at, build_duration: build.duration, @@ -66,8 +67,6 @@ module Gitlab environment: build_environment(build) } - - data end private @@ -84,7 +83,6 @@ module Gitlab id: runner.id, description: runner.description, active: runner.active?, - is_shared: runner.instance_type?, tags: runner.tags&.map(&:name) } end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 7fd1b9cd228..a56029c0d1d 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -77,7 +77,6 @@ module Gitlab id: runner.id, description: runner.description, active: runner.active?, - is_shared: runner.instance_type?, tags: runner.tags&.map(&:name) } end diff --git a/lib/gitlab/database/as_with_materialized.rb b/lib/gitlab/database/as_with_materialized.rb new file mode 100644 index 00000000000..7c45f416638 --- /dev/null +++ b/lib/gitlab/database/as_with_materialized.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # This class is a special Arel node which allows optionally define the `MATERIALIZED` keyword for CTE and Recursive CTE queries. + class AsWithMaterialized < Arel::Nodes::Binary + extend Gitlab::Utils::StrongMemoize + + MATERIALIZED = Arel.sql(' MATERIALIZED') + EMPTY_STRING = Arel.sql('') + attr_reader :expr + + def initialize(left, right, materialized: true) + @expr = if materialized && self.class.materialized_supported? + MATERIALIZED + else + EMPTY_STRING + end + + super(left, right) + end + + # Note: to be deleted after the minimum PG version is set to 12.0 + def self.materialized_supported? + strong_memoize(:materialized_supported) do + Gitlab::Database.version.match?(/^1[2-9]\./) # version 12.x and above + end + end + + # Note: to be deleted after the minimum PG version is set to 12.0 + def self.materialized_if_supported + materialized_supported? ? 'MATERIALIZED' : '' + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batch_metrics.rb b/lib/gitlab/database/background_migration/batch_metrics.rb new file mode 100644 index 00000000000..3e6d7ac3c9f --- /dev/null +++ b/lib/gitlab/database/background_migration/batch_metrics.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class BatchMetrics + attr_reader :timings + + def initialize + @timings = {} + end + + def time_operation(label) + start_time = monotonic_time + + yield + + timings_for_label(label) << monotonic_time - start_time + end + + private + + def timings_for_label(label) + timings[label] ||= [] + end + + def monotonic_time + Gitlab::Metrics::System.monotonic_time + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 0c9add9b355..4aa33ed7946 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -5,7 +5,7 @@ module Gitlab module BackgroundMigration class BatchedMigration < ActiveRecord::Base # rubocop:disable Rails/ApplicationRecord JOB_CLASS_MODULE = 'Gitlab::BackgroundMigration' - BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies".freeze + BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies" self.table_name = :batched_background_migrations @@ -23,8 +23,15 @@ module Gitlab finished: 3 } - def interval_elapsed? - last_job.nil? || last_job.created_at <= Time.current - interval + def self.active_migration + active.queue_order.first + end + + def interval_elapsed?(variance: 0) + return true unless last_job + + interval_with_variance = interval - variance + last_job.created_at <= Time.current - interval_with_variance end def create_batched_job!(min, max) @@ -50,6 +57,13 @@ module Gitlab def batch_class_name=(class_name) write_attribute(:batch_class_name, class_name.demodulize) end + + def prometheus_labels + @prometheus_labels ||= { + migration_id: id, + migration_identifier: "%s/%s.%s" % [job_class_name, table_name, column_name] + } + end end end end diff --git a/lib/gitlab/database/background_migration/scheduler.rb b/lib/gitlab/database/background_migration/batched_migration_runner.rb index 5f8a5ec06a5..cf8b61f5feb 100644 --- a/lib/gitlab/database/background_migration/scheduler.rb +++ b/lib/gitlab/database/background_migration/batched_migration_runner.rb @@ -3,12 +3,22 @@ module Gitlab module Database module BackgroundMigration - class Scheduler - def perform(migration_wrapper: BatchedMigrationWrapper.new) - active_migration = BatchedMigration.active.queue_order.first - - return unless active_migration&.interval_elapsed? + class BatchedMigrationRunner + def initialize(migration_wrapper = BatchedMigrationWrapper.new) + @migration_wrapper = migration_wrapper + end + # Runs the next batched_job for a batched_background_migration. + # + # The batch bounds of the next job are calculated at runtime, based on the migration + # configuration and the bounds of the most recently created batched_job. Updating the + # migration configuration will cause future jobs to use the updated batch sizes. + # + # The job instance will automatically receive a set of arguments based on the migration + # configuration. For more details, see the BatchedMigrationWrapper class. + # + # Note that this method is primarily intended to called by a scheduled worker. + def run_migration_job(active_migration) if next_batched_job = create_next_batched_job!(active_migration) migration_wrapper.perform(next_batched_job) else @@ -16,8 +26,26 @@ module Gitlab end end + # Runs all remaining batched_jobs for a batched_background_migration. + # + # This method is intended to be used in a test/dev environment to execute the background + # migration inline. It should NOT be used in a real environment for any non-trivial migrations. + def run_entire_migration(migration) + unless Rails.env.development? || Rails.env.test? + raise 'this method is not intended for use in real environments' + end + + while migration.active? + run_migration_job(migration) + + migration.reload_last_job + end + end + private + attr_reader :migration_wrapper + def create_next_batched_job!(active_migration) next_batch_range = find_next_batch_range(active_migration) diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index 299bd992197..c276f8ce75b 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -4,6 +4,15 @@ module Gitlab module Database module BackgroundMigration class BatchedMigrationWrapper + extend Gitlab::Utils::StrongMemoize + + # Wraps the execution of a batched_background_migration. + # + # Updates the job's tracking records with the status of the migration + # when starting and finishing execution, and optionally saves batch_metrics + # the migration provides, if any are given. + # + # The job's batch_metrics are serialized to JSON for storage. def perform(batch_tracking_record) start_tracking_execution(batch_tracking_record) @@ -16,6 +25,7 @@ module Gitlab raise e ensure finish_tracking_execution(batch_tracking_record) + track_prometheus_metrics(batch_tracking_record) end private @@ -34,12 +44,75 @@ module Gitlab tracking_record.migration_column_name, tracking_record.sub_batch_size, *tracking_record.migration_job_arguments) + + if job_instance.respond_to?(:batch_metrics) + tracking_record.metrics = job_instance.batch_metrics + end end def finish_tracking_execution(tracking_record) tracking_record.finished_at = Time.current tracking_record.save! end + + def track_prometheus_metrics(tracking_record) + migration = tracking_record.batched_migration + base_labels = migration.prometheus_labels + + metric_for(:gauge_batch_size).set(base_labels, tracking_record.batch_size) + metric_for(:gauge_sub_batch_size).set(base_labels, tracking_record.sub_batch_size) + metric_for(:counter_updated_tuples).increment(base_labels, tracking_record.batch_size) + + # Time efficiency: Ratio of duration to interval (ideal: less than, but close to 1) + efficiency = (tracking_record.finished_at - tracking_record.started_at).to_i / migration.interval.to_f + metric_for(:histogram_time_efficiency).observe(base_labels, efficiency) + + if metrics = tracking_record.metrics + metrics['timings']&.each do |key, timings| + summary = metric_for(:histogram_timings) + labels = base_labels.merge(operation: key) + + timings.each do |timing| + summary.observe(labels, timing) + end + end + end + end + + def metric_for(name) + self.class.metrics[name] + end + + def self.metrics + strong_memoize(:metrics) do + { + gauge_batch_size: Gitlab::Metrics.gauge( + :batched_migration_job_batch_size, + 'Batch size for a batched migration job' + ), + gauge_sub_batch_size: Gitlab::Metrics.gauge( + :batched_migration_job_sub_batch_size, + 'Sub-batch size for a batched migration job' + ), + counter_updated_tuples: Gitlab::Metrics.counter( + :batched_migration_job_updated_tuples_total, + 'Number of tuples updated by batched migration job' + ), + histogram_timings: Gitlab::Metrics.histogram( + :batched_migration_job_duration_seconds, + 'Timings for a batched migration job', + {}, + [0.1, 0.25, 0.5, 1, 5].freeze + ), + histogram_time_efficiency: Gitlab::Metrics.histogram( + :batched_migration_job_time_efficiency, + 'Ratio of job duration to interval', + {}, + [0.5, 0.9, 1, 1.5, 2].freeze + ) + } + end + end end end end diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 5a506da0d05..9002d39e1ee 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -88,11 +88,16 @@ module Gitlab batch_start = start while batch_start < finish - batch_end = [batch_start + batch_size, finish].min - batch_relation = build_relation_batch(batch_start, batch_end, mode) - begin - results = merge_results(results, batch_relation.send(@operation, *@operation_args)) # rubocop:disable GitlabSecurity/PublicSend + batch_end = [batch_start + batch_size, finish].min + batch_relation = build_relation_batch(batch_start, batch_end, mode) + + op_args = @operation_args + if @operation == :count && @operation_args.blank? && use_loose_index_scan_for_distinct_values?(mode) + op_args = [Gitlab::Database::LooseIndexScanDistinctCount::COLUMN_ALIAS] + end + + results = merge_results(results, batch_relation.send(@operation, *op_args)) # rubocop:disable GitlabSecurity/PublicSend batch_start = batch_end rescue ActiveRecord::QueryCanceled => error # retry with a safe batch size & warmer cache @@ -102,6 +107,18 @@ module Gitlab log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) return FALLBACK end + rescue Gitlab::Database::LooseIndexScanDistinctCount::ColumnConfigurationError => error + Gitlab::AppJsonLogger + .error( + event: 'batch_count', + relation: @relation.table_name, + operation: @operation, + operation_args: @operation_args, + mode: mode, + message: "LooseIndexScanDistinctCount column error: #{error.message}" + ) + + return FALLBACK end sleep(SLEEP_TIME_IN_SECONDS) @@ -123,7 +140,11 @@ module Gitlab private def build_relation_batch(start, finish, mode) - @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend + if use_loose_index_scan_for_distinct_values?(mode) + Gitlab::Database::LooseIndexScanDistinctCount.new(@relation, @column).build_query(from: start, to: finish) + else + @relation.select(@column).public_send(mode).where(between_condition(start, finish)) # rubocop:disable GitlabSecurity/PublicSend + end end def batch_size_for_mode_and_operation(mode, operation) @@ -165,6 +186,14 @@ module Gitlab message: "Query has been canceled with message: #{error.message}" ) end + + def use_loose_index_scan_for_distinct_values?(mode) + Feature.enabled?(:loose_index_scan_for_distinct_values) && not_group_by_query? && mode == :distinct + end + + def not_group_by_query? + !@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank? + end end end end diff --git a/lib/gitlab/database/bulk_update.rb b/lib/gitlab/database/bulk_update.rb index 1403d561890..b1f9da30585 100644 --- a/lib/gitlab/database/bulk_update.rb +++ b/lib/gitlab/database/bulk_update.rb @@ -130,7 +130,7 @@ module Gitlab def sql <<~SQL - WITH cte(#{list_of(cte_columns)}) AS (VALUES #{list_of(values)}) + WITH cte(#{list_of(cte_columns)}) AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (VALUES #{list_of(values)}) UPDATE #{table_name} SET #{list_of(updates)} FROM cte WHERE cte_id = id SQL end diff --git a/lib/gitlab/database/count/reltuples_count_strategy.rb b/lib/gitlab/database/count/reltuples_count_strategy.rb index 89190320cf9..a7bfafe2815 100644 --- a/lib/gitlab/database/count/reltuples_count_strategy.rb +++ b/lib/gitlab/database/count/reltuples_count_strategy.rb @@ -3,10 +3,6 @@ module Gitlab module Database module Count - class PgClass < ActiveRecord::Base - self.table_name = 'pg_class' - end - # This strategy counts based on PostgreSQL's statistics in pg_stat_user_tables. # # Specifically, it relies on the column reltuples in said table. An additional @@ -74,7 +70,7 @@ module Gitlab def get_statistics(table_names, check_statistics: true) time = 6.hours.ago - query = PgClass.joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid") + query = ::Gitlab::Database::PgClass.joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid") .where(relname: table_names) .where('schemaname = current_schema()') .select('pg_class.relname AS table_name, reltuples::bigint AS estimate') diff --git a/lib/gitlab/database/loose_index_scan_distinct_count.rb b/lib/gitlab/database/loose_index_scan_distinct_count.rb new file mode 100644 index 00000000000..884f4d47ff8 --- /dev/null +++ b/lib/gitlab/database/loose_index_scan_distinct_count.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # This class builds efficient batched distinct query by using loose index scan. + # Consider the following example: + # > Issue.distinct(:project_id).where(project_id: (1...100)).count + # + # Note: there is an index on project_id + # + # This query will read each element in the index matching the project_id filter. + # If for a project_id has 100_000 issues, all 100_000 elements will be read. + # + # A loose index scan will read only one entry from the index for each project_id to reduce the number of disk reads. + # + # Usage: + # + # Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).count(from: 1, to: 100) + # + # The query will return the number of distinct projects_ids between 1 and 100 + # + # Getting the Arel query: + # + # Gitlab::Database::LooseIndexScanDisctinctCount.new(Issue, :project_id).build_query(from: 1, to: 100) + class LooseIndexScanDistinctCount + COLUMN_ALIAS = 'distinct_count_column' + + ColumnConfigurationError = Class.new(StandardError) + + def initialize(scope, column) + if scope.is_a?(ActiveRecord::Relation) + @scope = scope + @model = scope.model + else + @scope = scope.where({}) + @model = scope + end + + @column = transform_column(column) + end + + def count(from:, to:) + build_query(from: from, to: to).count(COLUMN_ALIAS) + end + + def build_query(from:, to:) # rubocop:disable Metrics/AbcSize + cte = Gitlab::SQL::RecursiveCTE.new(:counter_cte, union_args: { remove_order: false }) + table = model.arel_table + + cte << @scope + .dup + .select(column.as(COLUMN_ALIAS)) + .where(column.gteq(from)) + .where(column.lt(to)) + .order(column) + .limit(1) + + inner_query = @scope + .dup + .where(column.gt(cte.table[COLUMN_ALIAS])) + .where(column.lt(to)) + .select(column.as(COLUMN_ALIAS)) + .order(column) + .limit(1) + + cte << cte.table + .project(Arel::Nodes::Grouping.new(Arel.sql(inner_query.to_sql)).as(COLUMN_ALIAS)) + .where(cte.table[COLUMN_ALIAS].lt(to)) + + model + .with + .recursive(cte.to_arel) + .from(cte.alias_to(table)) + .unscope(where: :source_type) + .unscope(where: model.inheritance_column) # Remove STI query, not needed here + end + + private + + attr_reader :column, :model + + # Transforms the column so it can be used in Arel expressions + # + # 'table.column' => 'table.column' + # 'column' => 'table_name.column' + # :column => 'table_name.column' + # Arel::Attributes::Attribute => name of the column + def transform_column(column) + if column.is_a?(String) || column.is_a?(Symbol) + column_as_string = column.to_s + column_as_string = "#{model.table_name}.#{column_as_string}" unless column_as_string.include?('.') + + Arel.sql(column_as_string) + elsif column.is_a?(Arel::Attributes::Attribute) + column + else + raise ColumnConfigurationError.new("Cannot transform the column: #{column.inspect}, please provide the column name as string") + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 31e733050e1..d06a73da8ac 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -4,6 +4,7 @@ module Gitlab module Database module MigrationHelpers include Migrations::BackgroundMigrationHelpers + include DynamicModelHelpers # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 @@ -576,17 +577,7 @@ module Gitlab # old_column - The name of the old column. # new_column - The name of the new column. def install_rename_triggers(table, old_column, new_column) - trigger_name = rename_trigger_name(table, old_column, new_column) - quoted_table = quote_table_name(table) - quoted_old = quote_column_name(old_column) - quoted_new = quote_column_name(new_column) - - install_rename_triggers_for_postgresql( - trigger_name, - quoted_table, - quoted_old, - quoted_new - ) + install_rename_triggers_for_postgresql(table, old_column, new_column) end # Changes the type of a column concurrently. @@ -927,19 +918,67 @@ module Gitlab # This is crucial for Primary Key conversions, because setting a column # as the PK converts even check constraints to NOT NULL constraints # and forces an inline re-verification of the whole table. - # - It backfills the new column with the values of the existing primary key - # by scheduling background jobs. - # - It tracks the scheduled background jobs through the use of - # Gitlab::Database::BackgroundMigrationJob + # - It sets up a trigger to keep the two columns in sync. + # + # Note: this helper is intended to be used in a regular (pre-deployment) migration. + # + # This helper is part 1 of a multi-step migration process: + # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers + # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations + # 3. remaining steps TBD, see #288005 + # + # table - The name of the database table containing the column + # column - The name of the column that we want to convert to bigint. + # primary_key - The name of the primary key column (most often :id) + def initialize_conversion_of_integer_to_bigint(table, column, primary_key: :id) + unless table_exists?(table) + raise "Table #{table} does not exist" + end + + unless column_exists?(table, primary_key) + raise "Column #{primary_key} does not exist on #{table}" + end + + unless column_exists?(table, column) + raise "Column #{column} does not exist on #{table}" + end + + check_trigger_permissions!(table) + + old_column = column_for(table, column) + tmp_column = "#{column}_convert_to_bigint" + + with_lock_retries do + if (column.to_s == primary_key.to_s) || !old_column.null + # If the column to be converted is either a PK or is defined as NOT NULL, + # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow + # That way, we skip the expensive validation step required to add + # a NOT NULL constraint at the end of the process + add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false) + else + add_column(table, tmp_column, :bigint, default: old_column.default) + end + + install_rename_triggers(table, column, tmp_column) + end + end + + # Backfills the new column used in the conversion of an integer column to bigint using background migrations. + # + # - This helper should be called from a post-deployment migration. + # - In order for this helper to work properly, the new column must be first initialized with + # the `initialize_conversion_of_integer_to_bigint` helper. + # - It tracks the scheduled background jobs through Gitlab::Database::BackgroundMigration::BatchedMigration, # which allows a more thorough check that all jobs succeeded in the # cleanup migration and is way faster for very large tables. - # - It sets up a trigger to keep the two columns in sync - # - It does not schedule a cleanup job: we have to do that with followup - # post deployment migrations in the next release. # - # This needs to be done manually by using the - # `cleanup_initialize_conversion_of_integer_to_bigint` - # (not yet implemented - check #288005) + # Note: this helper is intended to be used in a post-deployment migration, to ensure any new code is + # deployed (including background job changes) before we begin processing the background migration. + # + # This helper is part 2 of a multi-step migration process: + # 1. initialize_conversion_of_integer_to_bigint to create the new column and database triggers + # 2. backfill_conversion_of_integer_to_bigint to copy historic data using background migrations + # 3. remaining steps TBD, see #288005 # # table - The name of the database table containing the column # column - The name of the column that we want to convert to bigint. @@ -960,7 +999,7 @@ module Gitlab # and set the batch_size to 50_000 which will require # ~50s = (50000 / 200) * (0.1 + 0.1) to complete and leaves breathing space # between the scheduled jobs - def initialize_conversion_of_integer_to_bigint( + def backfill_conversion_of_integer_to_bigint( table, column, primary_key: :id, @@ -969,10 +1008,6 @@ module Gitlab interval: 2.minutes ) - if transaction_open? - raise 'initialize_conversion_of_integer_to_bigint can not be run inside a transaction' - end - unless table_exists?(table) raise "Table #{table} does not exist" end @@ -985,87 +1020,42 @@ module Gitlab raise "Column #{column} does not exist on #{table}" end - check_trigger_permissions!(table) - - old_column = column_for(table, column) tmp_column = "#{column}_convert_to_bigint" - with_lock_retries do - if (column.to_s == primary_key.to_s) || !old_column.null - # If the column to be converted is either a PK or is defined as NOT NULL, - # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow - # That way, we skip the expensive validation step required to add - # a NOT NULL constraint at the end of the process - add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false) - else - add_column(table, tmp_column, :bigint, default: old_column.default) - end - - install_rename_triggers(table, column, tmp_column) - end - - source_model = Class.new(ActiveRecord::Base) do - include EachBatch - - self.table_name = table - self.inheritance_column = :_type_disabled + unless column_exists?(table, tmp_column) + raise 'The temporary column does not exist, initialize it with `initialize_conversion_of_integer_to_bigint`' end - queue_background_migration_jobs_by_range_at_intervals( - source_model, + batched_migration = queue_batched_background_migration( 'CopyColumnUsingBackgroundMigrationJob', - interval, + table, + primary_key, + column, + tmp_column, + job_interval: interval, batch_size: batch_size, - other_job_arguments: [table, primary_key, sub_batch_size, column, tmp_column], - track_jobs: true, - primary_column_name: primary_key - ) + sub_batch_size: sub_batch_size) if perform_background_migration_inline? # To ensure the schema is up to date immediately we perform the # migration inline in dev / test environments. - Gitlab::BackgroundMigration.steal('CopyColumnUsingBackgroundMigrationJob') + Gitlab::Database::BackgroundMigration::BatchedMigrationRunner.new.run_entire_migration(batched_migration) end end # Performs a concurrent column rename when using PostgreSQL. - def install_rename_triggers_for_postgresql(trigger, table, old, new) - execute <<-EOF.strip_heredoc - CREATE OR REPLACE FUNCTION #{trigger}() - RETURNS trigger AS - $BODY$ - BEGIN - NEW.#{new} := NEW.#{old}; - RETURN NEW; - END; - $BODY$ - LANGUAGE 'plpgsql' - VOLATILE - EOF - - execute <<-EOF.strip_heredoc - DROP TRIGGER IF EXISTS #{trigger} - ON #{table} - EOF - - execute <<-EOF.strip_heredoc - CREATE TRIGGER #{trigger} - BEFORE INSERT OR UPDATE - ON #{table} - FOR EACH ROW - EXECUTE FUNCTION #{trigger}() - EOF + def install_rename_triggers_for_postgresql(table, old, new, trigger_name: nil) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).create(old, new, trigger_name: trigger_name) end # Removes the triggers used for renaming a PostgreSQL column concurrently. def remove_rename_triggers_for_postgresql(table, trigger) - execute("DROP TRIGGER IF EXISTS #{trigger} ON #{table}") - execute("DROP FUNCTION IF EXISTS #{trigger}()") + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).drop(trigger) end # Returns the (base) name to use for triggers when renaming columns. def rename_trigger_name(table, old, new) - 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old}_#{new}").first(12) + Gitlab::Database::UnidirectionalCopyTrigger.on_table(table).name(old, new) end # Returns an Array containing the indexes for the given column diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index e8cbea72887..8d5ea652bfc 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -190,7 +190,7 @@ module Gitlab migration_status = batch_max_value.nil? ? :finished : :active batch_max_value ||= batch_min_value - Gitlab::Database::BackgroundMigration::BatchedMigration.create!( + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.create!( job_class_name: job_class_name, table_name: batch_table_name, column_name: batch_column_name, @@ -202,6 +202,17 @@ module Gitlab sub_batch_size: sub_batch_size, job_arguments: job_arguments, status: migration_status) + + # This guard is necessary since #total_tuple_count was only introduced schema-wise, + # after this migration helper had been used for the first time. + return migration unless migration.respond_to?(:total_tuple_count) + + # We keep track of the estimated number of tuples to reason later + # about the overall progress of a migration. + migration.total_tuple_count = Gitlab::Database::PgClass.for_table(batch_table_name)&.cardinality_estimate + migration.save! + + migration end def perform_background_migration_inline? @@ -236,6 +247,14 @@ module Gitlab Gitlab::ApplicationContext.with_context(caller_id: self.class.to_s, &block) end + def delete_queued_jobs(class_name) + Gitlab::BackgroundMigration.steal(class_name) do |job| + job.delete + + false + end + end + private def track_in_database(class_name, arguments) diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb index 2def3a4d3a9..4402c42b136 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb @@ -6,6 +6,80 @@ module Gitlab module ForeignKeyHelpers include ::Gitlab::Database::SchemaHelpers + # Adds a foreign key with only minimal locking on the tables involved. + # + # In concept it works similarly to add_concurrent_foreign_key, but we have + # to add a special helper for partitioned tables for the following reasons: + # - add_concurrent_foreign_key sets the constraint to `NOT VALID` + # before validating it + # - Setting an FK to NOT VALID is not supported currently in Postgres (up to PG13) + # - Also, PostgreSQL will currently ignore NOT VALID constraints on partitions + # when adding a valid FK to the partitioned table, so they have to + # also be validated before we can add the final FK. + # Solution: + # - Add the foreign key first to each partition by using + # add_concurrent_foreign_key and validating it + # - Once all partitions have a foreign key, add it also to the partitioned + # table (there will be no need for a validation at that level) + # For those reasons, this method does not include an option to delay the + # validation, we have to force validate: true. + # + # source - The source (partitioned) table containing the foreign key. + # target - The target table the key points to. + # column - The name of the column to create the foreign key on. + # on_delete - The action to perform when associated data is removed, + # defaults to "CASCADE". + # name - The name of the foreign key. + # + def add_concurrent_partitioned_foreign_key(source, target, column:, on_delete: :cascade, name: nil) + partition_options = { + column: column, + on_delete: on_delete, + + # We'll use the same FK name for all partitions and match it to + # the name used for the partitioned table to follow the convention + # used by PostgreSQL when adding FKs to new partitions + name: name.presence || concurrent_partitioned_foreign_key_name(source, column), + + # Force the FK validation to true for partitions (and the partitioned table) + validate: true + } + + if foreign_key_exists?(source, target, **partition_options) + warning_message = "Foreign key not created because it exists already " \ + "(this may be due to an aborted migration or similar): " \ + "source: #{source}, target: #{target}, column: #{partition_options[:column]}, "\ + "name: #{partition_options[:name]}, on_delete: #{partition_options[:on_delete]}" + + Gitlab::AppLogger.warn warning_message + + return + end + + partitioned_table = find_partitioned_table(source) + + partitioned_table.postgres_partitions.order(:name).each do |partition| + add_concurrent_foreign_key(partition.identifier, target, **partition_options) + end + + with_lock_retries do + add_foreign_key(source, target, **partition_options) + end + end + + # Returns the name for a concurrent partitioned foreign key. + # + # Similar to concurrent_foreign_key_name (Gitlab::Database::MigrationHelpers) + # we just keep a separate method in case we want a different behavior + # for partitioned tables + # + def concurrent_partitioned_foreign_key_name(table, column, prefix: 'fk_rails_') + identifier = "#{table}_#{column}_fk" + hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) + + "#{prefix}#{hashed_identifier}" + end + # Creates a "foreign key" that references a partitioned table. Because foreign keys referencing partitioned # tables are not supported in PG11, this does not create a true database foreign key, but instead implements the # same functionality at the database level by using triggers. diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 1c289391e21..9ccbdc9930e 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -223,6 +223,28 @@ module Gitlab replace_table(table_name, archived_table_name, partitioned_table_name, primary_key_name) end + def drop_nonpartitioned_archive_table(table_name) + assert_table_is_allowed(table_name) + + archived_table_name = make_archived_table_name(table_name) + + with_lock_retries do + drop_sync_trigger(table_name) + end + + drop_table(archived_table_name) + end + + def create_trigger_to_sync_tables(source_table_name, partitioned_table_name, unique_key) + function_name = make_sync_function_name(source_table_name) + trigger_name = make_sync_trigger_name(source_table_name) + + create_sync_function(function_name, partitioned_table_name, unique_key) + create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table") + + create_sync_trigger(source_table_name, trigger_name, function_name) + end + private def assert_table_is_allowed(table_name) @@ -316,16 +338,6 @@ module Gitlab create_range_partition(partition_name, table_name, lower_bound, upper_bound) end - def create_trigger_to_sync_tables(source_table_name, partitioned_table_name, unique_key) - function_name = make_sync_function_name(source_table_name) - trigger_name = make_sync_trigger_name(source_table_name) - - create_sync_function(function_name, partitioned_table_name, unique_key) - create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table") - - create_sync_trigger(source_table_name, trigger_name, function_name) - end - def drop_sync_trigger(source_table_name) trigger_name = make_sync_trigger_name(source_table_name) drop_trigger(source_table_name, trigger_name) diff --git a/lib/gitlab/database/pg_class.rb b/lib/gitlab/database/pg_class.rb new file mode 100644 index 00000000000..0ce9eebc14c --- /dev/null +++ b/lib/gitlab/database/pg_class.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PgClass < ActiveRecord::Base + self.table_name = 'pg_class' + + def self.for_table(relname) + joins("LEFT JOIN pg_stat_user_tables ON pg_stat_user_tables.relid = pg_class.oid") + .where('schemaname = current_schema()') + .find_by(relname: relname) + end + + def cardinality_estimate + tuples = reltuples.to_i + + return if tuples < 1 + + tuples + end + end + end +end diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb index 62dfaeeaae3..e8b49c7f62c 100644 --- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb +++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb @@ -41,19 +41,6 @@ module Gitlab BUCKET_ID_MASK = (Buckets::TOTAL_BUCKETS - ZERO_OFFSET).to_s(2) BIT_31_MASK = "B'0#{'1' * 31}'" BIT_32_NORMALIZED_BUCKET_ID_MASK = "B'#{'0' * (32 - BUCKET_ID_MASK.size)}#{BUCKET_ID_MASK}'" - # @example source_query - # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits - # FROM %{relation} - # WHERE %{pkey} >= %{batch_start} - # AND %{pkey} < %{batch_end} - # AND %{column} IS NOT NULL - BUCKETED_DATA_SQL = <<~SQL - WITH hashed_attributes AS (%{source_query}) - SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num, - (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash - FROM hashed_attributes - GROUP BY 1 - SQL WRONG_CONFIGURATION_ERROR = Class.new(ActiveRecord::StatementInvalid) @@ -103,7 +90,7 @@ module Gitlab def hll_buckets_for_batch(start, finish) @relation .connection - .execute(BUCKETED_DATA_SQL % { source_query: source_query(start, finish) }) + .execute(bucketed_data_sql % { source_query: source_query(start, finish) }) .map(&:values) .to_h end @@ -139,6 +126,22 @@ module Gitlab def actual_finish(finish) finish || @relation.unscope(:group, :having).maximum(@relation.primary_key) || 0 end + + # @example source_query + # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits + # FROM %{relation} + # WHERE %{pkey} >= %{batch_start} + # AND %{pkey} < %{batch_end} + # AND %{column} IS NOT NULL + def bucketed_data_sql + <<~SQL + WITH hashed_attributes AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (%{source_query}) + SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num, + (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash + FROM hashed_attributes + GROUP BY 1 + SQL + end end end end diff --git a/lib/gitlab/database/similarity_score.rb b/lib/gitlab/database/similarity_score.rb index 40845c0d5e0..20bf6fa4d30 100644 --- a/lib/gitlab/database/similarity_score.rb +++ b/lib/gitlab/database/similarity_score.rb @@ -10,7 +10,7 @@ module Gitlab # Adds a "magic" comment in the generated SQL expression in order to be able to tell if we're sorting by similarity. # Example: /* gitlab/database/similarity_score */ SIMILARITY(COALESCE... - SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION = "/* #{DISPLAY_NAME} */ SIMILARITY".freeze + SIMILARITY_FUNCTION_CALL_WITH_ANNOTATION = "/* #{DISPLAY_NAME} */ SIMILARITY" # This method returns an Arel expression that can be used in an ActiveRecord query to order the resultset by similarity. # diff --git a/lib/gitlab/database/unidirectional_copy_trigger.rb b/lib/gitlab/database/unidirectional_copy_trigger.rb new file mode 100644 index 00000000000..029c894a5ff --- /dev/null +++ b/lib/gitlab/database/unidirectional_copy_trigger.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class UnidirectionalCopyTrigger + def self.on_table(table_name, connection: ActiveRecord::Base.connection) + new(table_name, connection) + end + + def name(from_column_names, to_column_names) + from_column_names, to_column_names = check_column_names!(from_column_names, to_column_names) + + unchecked_name(from_column_names, to_column_names) + end + + def create(from_column_names, to_column_names, trigger_name: nil) + from_column_names, to_column_names = check_column_names!(from_column_names, to_column_names) + trigger_name ||= unchecked_name(from_column_names, to_column_names) + + assignment_clauses = assignment_clauses_for_columns(from_column_names, to_column_names) + + connection.execute(<<~SQL) + CREATE OR REPLACE FUNCTION #{trigger_name}() + RETURNS trigger AS + $BODY$ + BEGIN + #{assignment_clauses}; + RETURN NEW; + END; + $BODY$ + LANGUAGE 'plpgsql' + VOLATILE + SQL + + connection.execute(<<~SQL) + DROP TRIGGER IF EXISTS #{trigger_name} + ON #{quoted_table_name} + SQL + + connection.execute(<<~SQL) + CREATE TRIGGER #{trigger_name} + BEFORE INSERT OR UPDATE + ON #{quoted_table_name} + FOR EACH ROW + EXECUTE FUNCTION #{trigger_name}() + SQL + end + + def drop(trigger_name) + connection.execute("DROP TRIGGER IF EXISTS #{trigger_name} ON #{quoted_table_name}") + connection.execute("DROP FUNCTION IF EXISTS #{trigger_name}()") + end + + private + + attr_reader :table_name, :connection + + def initialize(table_name, connection) + @table_name = table_name + @connection = connection + end + + def quoted_table_name + @quoted_table_name ||= connection.quote_table_name(table_name) + end + + def check_column_names!(from_column_names, to_column_names) + from_column_names = Array.wrap(from_column_names) + to_column_names = Array.wrap(to_column_names) + + unless from_column_names.size == to_column_names.size + raise ArgumentError, 'number of source and destination columns must match' + end + + [from_column_names, to_column_names] + end + + def unchecked_name(from_column_names, to_column_names) + joined_column_names = from_column_names.zip(to_column_names).flatten.join('_') + 'trigger_' + Digest::SHA256.hexdigest("#{table_name}_#{joined_column_names}").first(12) + end + + def assignment_clauses_for_columns(from_column_names, to_column_names) + combined_column_names = to_column_names.zip(from_column_names) + + assignment_clauses = combined_column_names.map do |(new_name, old_name)| + new_name = connection.quote_column_name(new_name) + old_name = connection.quote_column_name(old_name) + + "NEW.#{new_name} := NEW.#{old_name}" + end + + assignment_clauses.join(";\n ") + end + end + end +end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index baa46e7e306..8385bbbb3de 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,7 +3,7 @@ module Gitlab module Diff class Highlight - attr_reader :diff_file, :diff_lines, :raw_lines, :repository, :project + attr_reader :diff_file, :diff_lines, :repository, :project delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff @@ -22,29 +22,15 @@ module Gitlab end def highlight - @diff_lines.map.with_index do |diff_line, i| + populate_marker_ranges if Feature.enabled?(:use_marker_ranges, project, default_enabled: :yaml) + + @diff_lines.map.with_index do |diff_line, index| diff_line = diff_line.dup # ignore highlighting for "match" lines next diff_line if diff_line.meta? - rich_line = highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text) - - if line_inline_diffs = inline_diffs[i] - begin - # MarkerRange objects are converted to Ranges to keep the previous behavior - # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/324068 - if Feature.disabled?(:introduce_marker_ranges, project, default_enabled: :yaml) - line_inline_diffs = line_inline_diffs.map { |marker_range| marker_range.to_range } - end - - rich_line = InlineDiffMarker.new(diff_line.text, rich_line).mark(line_inline_diffs) - # This should only happen when the encoding of the diff doesn't - # match the blob, which is a bug. But we shouldn't fail to render - # completely in that case, even though we want to report the error. - rescue RangeError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/45441') - end - end + rich_line = apply_syntax_highlight(diff_line) + rich_line = apply_marker_ranges_highlight(diff_line, rich_line, index) diff_line.rich_text = rich_line @@ -54,9 +40,87 @@ module Gitlab private + def populate_marker_ranges + pair_selector = Gitlab::Diff::PairSelector.new(@raw_lines) + + pair_selector.each do |old_index, new_index| + old_line = diff_lines[old_index] + new_line = diff_lines[new_index] + + old_diffs, new_diffs = Gitlab::Diff::InlineDiff.new(old_line.text, new_line.text, offset: 1).inline_diffs + + old_line.set_marker_ranges(old_diffs) + new_line.set_marker_ranges(new_diffs) + end + end + + def apply_syntax_highlight(diff_line) + highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text) + end + + def apply_marker_ranges_highlight(diff_line, rich_line, index) + marker_ranges = if Feature.enabled?(:use_marker_ranges, project, default_enabled: :yaml) + diff_line.marker_ranges + else + inline_diffs[index] + end + + return rich_line if marker_ranges.blank? + + begin + # MarkerRange objects are converted to Ranges to keep the previous behavior + # Issue: https://gitlab.com/gitlab-org/gitlab/-/issues/324068 + if Feature.disabled?(:introduce_marker_ranges, project, default_enabled: :yaml) + marker_ranges = marker_ranges.map { |marker_range| marker_range.to_range } + end + + InlineDiffMarker.new(diff_line.text, rich_line).mark(marker_ranges) + # This should only happen when the encoding of the diff doesn't + # match the blob, which is a bug. But we shouldn't fail to render + # completely in that case, even though we want to report the error. + rescue RangeError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, issue_url: 'https://gitlab.com/gitlab-org/gitlab-foss/issues/45441') + end + end + def highlight_line(diff_line) return unless diff_file && diff_file.diff_refs + if Feature.enabled?(:diff_line_syntax_highlighting, project, default_enabled: :yaml) + diff_line_highlighting(diff_line) + else + blob_highlighting(diff_line) + end + end + + def diff_line_highlighting(diff_line) + rich_line = syntax_highlighter(diff_line).highlight( + diff_line.text(prefix: false), + context: { line_number: diff_line.line } + )&.html_safe + + # Only update text if line is found. This will prevent + # issues with submodules given the line only exists in diff content. + if rich_line + line_prefix = diff_line.text =~ /\A(.)/ ? Regexp.last_match(1) : ' ' + rich_line.prepend(line_prefix).concat("\n") + end + end + + def syntax_highlighter(diff_line) + path = diff_line.removed? ? diff_file.old_path : diff_file.new_path + + @syntax_highlighter ||= {} + @syntax_highlighter[path] ||= Gitlab::Highlight.new( + path, + @raw_lines, + language: repository&.gitattribute(path, 'gitlab-language') + ) + end + + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324159 + # ------------------------------------------------------------------------ + def blob_highlighting(diff_line) rich_line = if diff_line.unchanged? || diff_line.added? new_lines[diff_line.new_pos - 1]&.html_safe @@ -72,6 +136,8 @@ module Gitlab end end + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324638 + # ------------------------------------------------------------------------ def inline_diffs @inline_diffs ||= InlineDiff.for_lines(@raw_lines) end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index c5e9bfdc321..209462fd6e9 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -71,9 +71,12 @@ module Gitlab strong_memoize(:redis_key) do [ 'highlighted-diff-files', - diffable.cache_key, VERSION, + diffable.cache_key, + VERSION, diff_options, - Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml) + Feature.enabled?(:introduce_marker_ranges, diffable.project, default_enabled: :yaml), + Feature.enabled?(:use_marker_ranges, diffable.project, default_enabled: :yaml), + Feature.enabled?(:diff_line_syntax_highlighting, diffable.project, default_enabled: :yaml) ].join(":") end end diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index dd73e4d6c15..f70618195d0 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -18,6 +18,7 @@ module Gitlab CharDiff.new(old_line, new_line).changed_ranges(offset: offset) end + # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324638 class << self def for_lines(lines) pair_selector = Gitlab::Diff::PairSelector.new(lines) diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 98ed2400d82..6cf414e29cc 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -8,19 +8,24 @@ module Gitlab # SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze - attr_reader :line_code - attr_writer :rich_text - attr_accessor :text, :index, :type, :old_pos, :new_pos + attr_reader :line_code, :marker_ranges + attr_writer :text, :rich_text + attr_accessor :index, :type, :old_pos, :new_pos def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) - @text, @type, @index = text, type, index - @old_pos, @new_pos = old_pos, new_pos + @text = text + @type = type + @index = index + @old_pos = old_pos + @new_pos = new_pos @parent_file = parent_file @rich_text = rich_text # When line code is not provided from cache store we build it # using the parent_file(Diff::File or Conflict::File). @line_code = line_code || calculate_line_code + + @marker_ranges = [] end def self.init_from_hash(hash) @@ -48,6 +53,16 @@ module Gitlab hash end + def set_marker_ranges(marker_ranges) + @marker_ranges = marker_ranges + end + + def text(prefix: true) + return @text if prefix + + @text&.slice(1..).to_s + end + def old_line old_pos unless added? || meta? end diff --git a/lib/gitlab/diff/suggestions_parser.rb b/lib/gitlab/diff/suggestions_parser.rb index 6e17ffaf6ff..f3e6fc455ac 100644 --- a/lib/gitlab/diff/suggestions_parser.rb +++ b/lib/gitlab/diff/suggestions_parser.rb @@ -17,7 +17,7 @@ module Gitlab no_original_data: true, suggestions_filter_enabled: supports_suggestion) doc = Nokogiri::HTML(html) - suggestion_nodes = doc.search('pre.suggestion') + suggestion_nodes = doc.search('pre.language-suggestion') return [] if suggestion_nodes.empty? @@ -29,9 +29,8 @@ module Gitlab lines_above, lines_below = nil if lang_param && suggestion_params = fetch_suggestion_params(lang_param) - lines_above, lines_below = - suggestion_params[:above], - suggestion_params[:below] + lines_above = suggestion_params[:above] + lines_below = suggestion_params[:below] end Gitlab::Diff::Suggestion.new(node.text, diff --git a/lib/gitlab/downtime_check.rb b/lib/gitlab/downtime_check.rb deleted file mode 100644 index 457a3c12206..00000000000 --- a/lib/gitlab/downtime_check.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - # Checks if a set of migrations requires downtime or not. - class DowntimeCheck - # The constant containing the boolean that indicates if downtime is needed - # or not. - DOWNTIME_CONST = :DOWNTIME - - # The constant that specifies the reason for the migration requiring - # downtime. - DOWNTIME_REASON_CONST = :DOWNTIME_REASON - - # Checks the given migration paths and returns an Array of - # `Gitlab::DowntimeCheck::Message` instances. - # - # migrations - The migration file paths to check. - def check(migrations) - migrations.map do |path| - require(path) - - migration_class = class_for_migration_file(path) - - unless migration_class.const_defined?(DOWNTIME_CONST) - raise "The migration in #{path} does not specify if it requires " \ - "downtime or not" - end - - if online?(migration_class) - Message.new(path) - else - reason = downtime_reason(migration_class) - - unless reason - raise "The migration in #{path} requires downtime but no reason " \ - "was given" - end - - Message.new(path, true, reason) - end - end - end - - # Checks the given migrations and prints the results to STDOUT/STDERR. - # - # migrations - The migration file paths to check. - def check_and_print(migrations) - check(migrations).each do |message| - puts message.to_s # rubocop: disable Rails/Output - end - end - - # Returns the class for the given migration file path. - def class_for_migration_file(path) - File.basename(path, File.extname(path)).split('_', 2).last.camelize - .constantize - end - - # Returns true if the given migration can be performed without downtime. - def online?(migration) - migration.const_get(DOWNTIME_CONST, false) == false - end - - # Returns the downtime reason, or nil if none was defined. - def downtime_reason(migration) - if migration.const_defined?(DOWNTIME_REASON_CONST) - migration.const_get(DOWNTIME_REASON_CONST, false) - else - nil - end - end - end -end diff --git a/lib/gitlab/downtime_check/message.rb b/lib/gitlab/downtime_check/message.rb deleted file mode 100644 index 5debb754943..00000000000 --- a/lib/gitlab/downtime_check/message.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class DowntimeCheck - class Message - attr_reader :path, :offline - - OFFLINE = "\e[31moffline\e[0m" - ONLINE = "\e[32monline\e[0m" - - # path - The file path of the migration. - # offline - When set to `true` the migration will require downtime. - # reason - The reason as to why the migration requires downtime. - def initialize(path, offline = false, reason = nil) - @path = path - @offline = offline - @reason = reason - end - - def to_s - label = offline ? OFFLINE : ONLINE - - message = ["[#{label}]: #{path}"] - - if reason? - message << ":\n\n#{reason}\n\n" - end - - message.join - end - - def reason? - @reason.present? - end - - def reason - @reason.strip.lines.map(&:strip).join("\n") - end - end - end -end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index dfed8db8df0..47d361fb95c 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -16,6 +16,12 @@ module Gitlab Rack::Timeout::RequestTimeoutException ].freeze + PROCESSORS = [ + ::Gitlab::ErrorTracking::Processor::SidekiqProcessor, + ::Gitlab::ErrorTracking::Processor::GrpcErrorProcessor, + ::Gitlab::ErrorTracking::Processor::ContextPayloadProcessor + ].freeze + class << self def configure Raven.configure do |config| @@ -97,7 +103,9 @@ module Gitlab inject_context_for_exception(event, hint[:exception]) custom_fingerprinting(event, hint[:exception]) - event + PROCESSORS.reduce(event) do |processed_event, processor| + processor.call(processed_event) + end end def process_exception(exception, sentry: false, logging: true, extra:) diff --git a/lib/gitlab/error_tracking/processor/context_payload_processor.rb b/lib/gitlab/error_tracking/processor/context_payload_processor.rb index 5185205e94e..758f6aa11d7 100644 --- a/lib/gitlab/error_tracking/processor/context_payload_processor.rb +++ b/lib/gitlab/error_tracking/processor/context_payload_processor.rb @@ -9,9 +9,21 @@ module Gitlab # integrations are re-implemented and use Gitlab::ErrorTracking, this # processor should be removed. def process(payload) + return payload if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) + context_payload = Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {}) payload.deep_merge!(context_payload) end + + def self.call(event) + return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) + + Gitlab::ErrorTracking::ContextPayloadGenerator.generate(nil, {}).each do |key, value| + event.public_send(key).deep_merge!(value) # rubocop:disable GitlabSecurity/PublicSend + end + + event + end end end end diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb index 871e9c4b7c8..419098dbd09 100644 --- a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb +++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb @@ -6,60 +6,126 @@ module Gitlab class GrpcErrorProcessor < ::Raven::Processor DEBUG_ERROR_STRING_REGEX = RE2('(.*) debug_error_string:(.*)') - def process(value) - process_first_exception_value(value) - process_custom_fingerprint(value) + def process(payload) + return payload if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) - value - end - - # Sentry can report multiple exceptions in an event. Sanitize - # only the first one since that's what is used for grouping. - def process_first_exception_value(value) - exceptions = value.dig(:exception, :values) - - return unless exceptions.is_a?(Array) - - entry = exceptions.first - - return unless entry.is_a?(Hash) - - exception_type = entry[:type] - raw_message = entry[:value] - - return unless exception_type&.start_with?('GRPC::') - return unless raw_message.present? - - message, debug_str = split_debug_error_string(raw_message) - - entry[:value] = message if message - extra = value[:extra] || {} - extra[:grpc_debug_error_string] = debug_str if debug_str - end - - def process_custom_fingerprint(value) - fingerprint = value[:fingerprint] - - return value unless custom_grpc_fingerprint?(fingerprint) + self.class.process_first_exception_value(payload) + self.class.process_custom_fingerprint(payload) - message, _ = split_debug_error_string(fingerprint[1]) - fingerprint[1] = message if message + payload end - private - - def custom_grpc_fingerprint?(fingerprint) - fingerprint.is_a?(Array) && fingerprint.length == 2 && fingerprint[0].start_with?('GRPC::') - end - - def split_debug_error_string(message) - return unless message - - match = DEBUG_ERROR_STRING_REGEX.match(message) - - return unless match - - [match[1], match[2]] + class << self + def call(event) + return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) + + process_first_exception_value(event) + process_custom_fingerprint(event) + + event + end + + # Sentry can report multiple exceptions in an event. Sanitize + # only the first one since that's what is used for grouping. + def process_first_exception_value(event_or_payload) + exceptions = exceptions(event_or_payload) + + return unless exceptions.is_a?(Array) + + exception = exceptions.first + + return unless valid_exception?(exception) + + exception_type, raw_message = type_and_value(exception) + + return unless exception_type&.start_with?('GRPC::') + return unless raw_message.present? + + message, debug_str = split_debug_error_string(raw_message) + + set_new_values!(event_or_payload, exception, message, debug_str) + end + + def process_custom_fingerprint(event) + fingerprint = fingerprint(event) + + return event unless custom_grpc_fingerprint?(fingerprint) + + message, _ = split_debug_error_string(fingerprint[1]) + fingerprint[1] = message if message + end + + private + + def custom_grpc_fingerprint?(fingerprint) + fingerprint.is_a?(Array) && fingerprint.length == 2 && fingerprint[0].start_with?('GRPC::') + end + + def split_debug_error_string(message) + return unless message + + match = DEBUG_ERROR_STRING_REGEX.match(message) + + return unless match + + [match[1], match[2]] + end + + # The below methods can be removed once we remove the + # sentry_processors_before_send feature flag, and we can + # assume we always have an Event object + def exceptions(event_or_payload) + case event_or_payload + when Raven::Event + # Better in new version, will be event_or_payload.exception.values + event_or_payload.instance_variable_get(:@interfaces)[:exception]&.values + when Hash + event_or_payload.dig(:exception, :values) + end + end + + def valid_exception?(exception) + case exception + when Raven::SingleExceptionInterface + exception&.value + when Hash + true + else + false + end + end + + def type_and_value(exception) + case exception + when Raven::SingleExceptionInterface + [exception.type, exception.value] + when Hash + exception.values_at(:type, :value) + end + end + + def set_new_values!(event_or_payload, exception, message, debug_str) + case event_or_payload + when Raven::Event + # Worse in new version, no setter! Have to poke at the + # instance variable + exception.value = message if message + event_or_payload.extra[:grpc_debug_error_string] = debug_str if debug_str + when Hash + exception[:value] = message if message + extra = event_or_payload[:extra] || {} + extra[:grpc_debug_error_string] = debug_str if debug_str + end + end + + def fingerprint(event_or_payload) + case event_or_payload + when Raven::Event + event_or_payload.fingerprint + when Hash + event_or_payload[:fingerprint] + end + end end end end diff --git a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb index 272cb689ad5..93310745ece 100644 --- a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb +++ b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb @@ -8,39 +8,66 @@ module Gitlab class SidekiqProcessor < ::Raven::Processor FILTERED_STRING = '[FILTERED]' - def self.filter_arguments(args, klass) - args.lazy.with_index.map do |arg, i| - case arg - when Numeric - arg - else - if permitted_arguments_for_worker(klass).include?(i) + class << self + def filter_arguments(args, klass) + args.lazy.with_index.map do |arg, i| + case arg + when Numeric arg else - FILTERED_STRING + if permitted_arguments_for_worker(klass).include?(i) + arg + else + FILTERED_STRING + end end end end - end - def self.permitted_arguments_for_worker(klass) - @permitted_arguments_for_worker ||= {} - @permitted_arguments_for_worker[klass] ||= - begin - klass.constantize&.loggable_arguments&.to_set - rescue - Set.new + def permitted_arguments_for_worker(klass) + @permitted_arguments_for_worker ||= {} + @permitted_arguments_for_worker[klass] ||= + begin + klass.constantize&.loggable_arguments&.to_set + rescue + Set.new + end + end + + def loggable_arguments(args, klass) + Gitlab::Utils::LogLimitedArray + .log_limited_array(filter_arguments(args, klass)) + .map(&:to_s) + .to_a + end + + def call(event) + return event unless ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) + + sidekiq = event&.extra&.dig(:sidekiq) + + return event unless sidekiq + + sidekiq = sidekiq.deep_dup + sidekiq.delete(:jobstr) + + # 'args' in this hash => from Gitlab::ErrorTracking.track_* + # 'args' in :job => from default error handler + job_holder = sidekiq.key?('args') ? sidekiq : sidekiq[:job] + + if job_holder['args'] + job_holder['args'] = filter_arguments(job_holder['args'], job_holder['class']).to_a end - end - def self.loggable_arguments(args, klass) - Gitlab::Utils::LogLimitedArray - .log_limited_array(filter_arguments(args, klass)) - .map(&:to_s) - .to_a + event.extra[:sidekiq] = sidekiq + + event + end end def process(value, key = nil) + return value if ::Feature.enabled?(:sentry_processors_before_send, default_enabled: :yaml) + sidekiq = value.dig(:extra, :sidekiq) return value unless sidekiq diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index b602393b59e..ef0236f8275 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -15,14 +15,14 @@ module Gitlab PREFIX = 'gitlab:exclusive_lease' NoKey = Class.new(ArgumentError) - LUA_CANCEL_SCRIPT = <<~EOS.freeze + LUA_CANCEL_SCRIPT = <<~EOS local key, uuid = KEYS[1], ARGV[1] if redis.call("get", key) == uuid then redis.call("del", key) end EOS - LUA_RENEW_SCRIPT = <<~EOS.freeze + LUA_RENEW_SCRIPT = <<~EOS local key, uuid, ttl = KEYS[1], ARGV[1], ARGV[2] if redis.call("get", key) == uuid then redis.call("expire", key, ttl) diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 1bb29ba3eac..145bb6d7b8f 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -34,10 +34,6 @@ module Gitlab module Experimentation EXPERIMENTS = { - upgrade_link_in_user_menu_a: { - tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA', - use_backwards_compatible_subject_index: true - }, invite_members_version_b: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionB', use_backwards_compatible_subject_index: true diff --git a/lib/gitlab/external_authorization/access.rb b/lib/gitlab/external_authorization/access.rb index e111c41fcc2..21fa728fd3a 100644 --- a/lib/gitlab/external_authorization/access.rb +++ b/lib/gitlab/external_authorization/access.rb @@ -10,7 +10,8 @@ module Gitlab :load_type def initialize(user, label) - @user, @label = user, label + @user = user + @label = label end def loaded? diff --git a/lib/gitlab/external_authorization/cache.rb b/lib/gitlab/external_authorization/cache.rb index acdc028b4dc..509daeb0248 100644 --- a/lib/gitlab/external_authorization/cache.rb +++ b/lib/gitlab/external_authorization/cache.rb @@ -6,7 +6,8 @@ module Gitlab VALIDITY_TIME = 6.hours def initialize(user, label) - @user, @label = user, label + @user = user + @label = label end def load diff --git a/lib/gitlab/external_authorization/client.rb b/lib/gitlab/external_authorization/client.rb index fc859304eab..582051010d3 100644 --- a/lib/gitlab/external_authorization/client.rb +++ b/lib/gitlab/external_authorization/client.rb @@ -13,7 +13,8 @@ module Gitlab }.freeze def initialize(user, label) - @user, @label = user, label + @user = user + @label = label end def request_access @@ -51,18 +52,18 @@ module Gitlab def body @body ||= begin - body = { - user_identifier: @user.email, - project_classification_label: @label, - identities: @user.identities.map { |identity| { provider: identity.provider, extern_uid: identity.extern_uid } } - } + body = { + user_identifier: @user.email, + project_classification_label: @label, + identities: @user.identities.map { |identity| { provider: identity.provider, extern_uid: identity.extern_uid } } + } - if @user.ldap_identity - body[:user_ldap_dn] = @user.ldap_identity.extern_uid - end + if @user.ldap_identity + body[:user_ldap_dn] = @user.ldap_identity.extern_uid + end - body - end + body + end end end end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index bd5d2e53180..612865ed1be 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -199,8 +199,7 @@ module Gitlab def linkify_issues(str) str = str.gsub(/([Ii]ssue) ([0-9]+)/, '\1 #\2') - str = str.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2') - str + str.gsub(/([Cc]ase) ([0-9]+)/, '\1 #\2') end def escape_for_markdown(str) @@ -208,8 +207,7 @@ module Gitlab str = str.gsub(/^-/, "\\-") str = str.gsub("`", "\\~") str = str.delete("\r") - str = str.gsub("\n", " \n") - str + str.gsub("\n", " \n") end def format_content(raw_content) diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 9e24306c05e..a5b1b7d914b 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -30,8 +30,10 @@ module Gitlab end def process_raw_blame(output) - lines, final = [], [] - info, commits = {}, {} + lines = [] + final = [] + info = {} + commits = {} # process the output output.split("\n").each do |line| diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index ff99803d8de..51baed32935 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -390,7 +390,7 @@ module Gitlab @committer_name = commit.committer.name.dup @committer_email = commit.committer.email.dup @parent_ids = Array(commit.parent_ids) - @trailers = Hash[commit.trailers.map { |t| [t.key, t.value] }] + @trailers = commit.trailers.to_h { |t| [t.key, t.value] } end # Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 19462e6cb02..fb947c80b7e 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -82,6 +82,30 @@ module Gitlab !!@overflow end + def overflow_max_lines? + !!@overflow_max_lines + end + + def overflow_max_bytes? + !!@overflow_max_bytes + end + + def overflow_max_files? + !!@overflow_max_files + end + + def collapsed_safe_lines? + !!@collapsed_safe_lines + end + + def collapsed_safe_files? + !!@collapsed_safe_files + end + + def collapsed_safe_bytes? + !!@collapsed_safe_bytes + end + def size @size ||= count # forces a loop using each method end @@ -103,10 +127,9 @@ module Gitlab end def decorate! - collection = each_with_index do |element, i| + each_with_index do |element, i| @array[i] = yield(element) end - collection end alias_method :to_ary, :to_a @@ -121,7 +144,15 @@ module Gitlab end def over_safe_limits?(files) - files >= safe_max_files || @line_count > safe_max_lines || @byte_count >= safe_max_bytes + if files >= safe_max_files + @collapsed_safe_files = true + elsif @line_count > safe_max_lines + @collapsed_safe_lines = true + elsif @byte_count >= safe_max_bytes + @collapsed_safe_bytes = true + end + + @collapsed_safe_files || @collapsed_safe_lines || @collapsed_safe_bytes end def expand_diff? @@ -154,6 +185,7 @@ module Gitlab if @enforce_limits && i >= max_files @overflow = true + @overflow_max_files = true break end @@ -166,10 +198,19 @@ module Gitlab @line_count += diff.line_count @byte_count += diff.diff.bytesize - if @enforce_limits && (@line_count >= max_lines || @byte_count >= max_bytes) + if @enforce_limits && @line_count >= max_lines + # This last Diff instance pushes us over the lines limit. We stop and + # discard it. + @overflow = true + @overflow_max_lines = true + break + end + + if @enforce_limits && @byte_count >= max_bytes # This last Diff instance pushes us over the lines limit. We stop and # discard it. @overflow = true + @overflow_max_bytes = true break end diff --git a/lib/gitlab/git/merge_base.rb b/lib/gitlab/git/merge_base.rb index b27f7038c26..905d72cadbf 100644 --- a/lib/gitlab/git/merge_base.rb +++ b/lib/gitlab/git/merge_base.rb @@ -6,7 +6,8 @@ module Gitlab include Gitlab::Utils::StrongMemoize def initialize(repository, refs) - @repository, @refs = repository, refs + @repository = repository + @refs = refs end # Returns the SHA of the first common ancestor diff --git a/lib/gitlab/git/patches/commit_patches.rb b/lib/gitlab/git/patches/commit_patches.rb index c62994432d3..1182db10c34 100644 --- a/lib/gitlab/git/patches/commit_patches.rb +++ b/lib/gitlab/git/patches/commit_patches.rb @@ -7,7 +7,10 @@ module Gitlab include Gitlab::Git::WrapsGitalyErrors def initialize(user, repository, branch, patch_collection) - @user, @repository, @branch, @patches = user, repository, branch, patch_collection + @user = user + @repository = repository + @branch = branch + @patches = patch_collection end def commit diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e316d52ac05..3361cee733b 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -599,9 +599,9 @@ module Gitlab tags.find { |tag| tag.name == name } end - def merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts) + def merge_to_ref(user, **kwargs) wrapped_gitaly_errors do - gitaly_operation_client.user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts) + gitaly_operation_client.user_merge_to_ref(user, **kwargs) end end @@ -1017,6 +1017,10 @@ module Gitlab gitaly_repository_client.search_files_by_name(ref, safe_query) end + def search_files_by_regexp(filter, ref = 'HEAD') + gitaly_repository_client.search_files_by_regexp(ref, filter) + end + def find_commits_by_message(query, ref, path, limit, offset) wrapped_gitaly_errors do gitaly_commit_client diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index da86d6baf4a..568e894a02f 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -87,6 +87,10 @@ module Gitlab end end + def cache_key + "tag:" + Digest::SHA1.hexdigest([name, message, target, target_commit&.sha].join) + end + private def message_from_gitaly_tag diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 55ff3c6caf1..75d6b949874 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -102,12 +102,6 @@ module Gitlab end end - def file(name, version) - wrapped_gitaly_errors do - gitaly_find_file(name, version) - end - end - # options: # :page - The Integer page number. # :per_page - The number of items per page. @@ -161,13 +155,6 @@ module Gitlab nil end - def gitaly_find_file(name, version) - wiki_file = gitaly_wiki_client.find_file(name, version) - return unless wiki_file - - Gitlab::Git::WikiFile.new(wiki_file) - end - def gitaly_list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false) params = { limit: limit, sort: sort, direction_desc: direction_desc } diff --git a/lib/gitlab/git/wiki_file.rb b/lib/gitlab/git/wiki_file.rb index 7f09173f05c..c56a17c52f3 100644 --- a/lib/gitlab/git/wiki_file.rb +++ b/lib/gitlab/git/wiki_file.rb @@ -5,25 +5,11 @@ module Gitlab class WikiFile attr_reader :mime_type, :raw_data, :name, :path - # This class wraps Gitlab::GitalyClient::WikiFile - def initialize(gitaly_file) - @mime_type = gitaly_file.mime_type - @raw_data = gitaly_file.raw_data - @name = gitaly_file.name - @path = gitaly_file.path - end - - def self.from_blob(blob) - hash = { - name: File.basename(blob.name), - mime_type: blob.mime_type, - path: blob.path, - raw_data: blob.data - } - - gitaly_file = Gitlab::GitalyClient::WikiFile.new(hash) - - Gitlab::Git::WikiFile.new(gitaly_file) + def initialize(blob) + @mime_type = blob.mime_type + @raw_data = blob.data + @name = File.basename(blob.name) + @path = blob.path end end end diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index c5ca46827cb..31e4755192e 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -91,6 +91,7 @@ module Gitlab when *PUSH_COMMANDS check_push_access! end + check_additional_conditions! success_result end @@ -530,6 +531,10 @@ module Gitlab def size_checker container.repository_size_checker end + + # overriden in EE + def check_additional_conditions! + end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index e3788814dd5..f4a89edecd1 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -215,7 +215,7 @@ module Gitlab 'client_name' => CLIENT_NAME } - context_data = Labkit::Context.current&.to_h + context_data = Gitlab::ApplicationContext.current feature_stack = Thread.current[:gitaly_feature_stack] feature = feature_stack && feature_stack[0] diff --git a/lib/gitlab/gitaly_client/attributes_bag.rb b/lib/gitlab/gitaly_client/attributes_bag.rb index f935281ac2e..74e6279708e 100644 --- a/lib/gitlab/gitaly_client/attributes_bag.rb +++ b/lib/gitlab/gitaly_client/attributes_bag.rb @@ -3,7 +3,7 @@ module Gitlab module GitalyClient # This module expects an `ATTRS` const to be defined on the subclass - # See GitalyClient::WikiFile for an example + # See GitalyClient::WikiPage for an example module AttributesBag extend ActiveSupport::Concern diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index c66b3335d89..19a473e4785 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -78,17 +78,7 @@ module Gitlab end def get_new_lfs_pointers(revision, limit, not_in, dynamic_timeout = nil) - request = Gitaly::GetNewLFSPointersRequest.new( - repository: @gitaly_repo, - revision: encode_binary(revision), - limit: limit || 0 - ) - - if not_in.nil? || not_in == :all - request.not_in_all = true - else - request.not_in_refs += not_in - end + request, rpc = create_new_lfs_pointers_request(revision, limit, not_in) timeout = if dynamic_timeout @@ -100,7 +90,7 @@ module Gitlab response = GitalyClient.call( @gitaly_repo.storage_name, :blob_service, - :get_new_lfs_pointers, + rpc, request, timeout: timeout ) @@ -108,16 +98,51 @@ module Gitlab end def get_all_lfs_pointers - request = Gitaly::GetAllLFSPointersRequest.new( - repository: @gitaly_repo + request = Gitaly::ListLFSPointersRequest.new( + repository: @gitaly_repo, + revisions: [encode_binary("--all")] ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request, timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :list_lfs_pointers, request, timeout: GitalyClient.medium_timeout) map_lfs_pointers(response) end private + def create_new_lfs_pointers_request(revision, limit, not_in) + # If the check happens for a change which is using a quarantine + # environment for incoming objects, then we can avoid doing the + # necessary graph walk to detect only new LFS pointers and instead scan + # through all quarantined objects. + git_env = ::Gitlab::Git::HookEnv.all(@gitaly_repo.gl_repository) + if Feature.enabled?(:lfs_integrity_inspect_quarantined_objects, @project, default_enabled: :yaml) && git_env['GIT_OBJECT_DIRECTORY_RELATIVE'].present? + repository = @gitaly_repo.dup + repository.git_alternate_object_directories = Google::Protobuf::RepeatedField.new(:string) + + request = Gitaly::ListAllLFSPointersRequest.new( + repository: repository, + limit: limit || 0 + ) + + [request, :list_all_lfs_pointers] + else + revisions = [revision] + revisions += if not_in.nil? || not_in == :all + ["--not", "--all"] + else + not_in.prepend "--not" + end + + request = Gitaly::ListLFSPointersRequest.new( + repository: @gitaly_repo, + limit: limit || 0, + revisions: revisions.map { |rev| encode_binary(rev) } + ) + + [request, :list_lfs_pointers] + end + end + def consume_blob_response(response) data = [] blob = nil diff --git a/lib/gitlab/gitaly_client/call.rb b/lib/gitlab/gitaly_client/call.rb index 9d4d86997ad..4bb184bee2f 100644 --- a/lib/gitlab/gitaly_client/call.rb +++ b/lib/gitlab/gitaly_client/call.rb @@ -50,11 +50,11 @@ module Gitlab end def recording_request - start = Gitlab::Metrics::System.monotonic_time + @start = Gitlab::Metrics::System.monotonic_time yield ensure - @duration += Gitlab::Metrics::System.monotonic_time - start + @duration += Gitlab::Metrics::System.monotonic_time - @start end def store_timings @@ -64,8 +64,14 @@ module Gitlab request_hash = @request.is_a?(Google::Protobuf::MessageExts) ? @request.to_h : {} - GitalyClient.add_call_details(feature: "#{@service}##{@rpc}", duration: @duration, request: request_hash, rpc: @rpc, - backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller)) + GitalyClient.add_call_details( + start: @start, + feature: "#{@service}##{@rpc}", + duration: @duration, + request: request_hash, + rpc: @rpc, + backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller) + ) end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index ef5221a8042..3d24b4d53a4 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -107,6 +107,8 @@ module Gitlab entry.data = data.join entry unless entry.oid.blank? + rescue GRPC::NotFound + nil end def tree_entries(repository, revision, path, recursive) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 6f302b2c4e7..5ce1b1f0c87 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -103,7 +103,7 @@ module Gitlab end end - def user_merge_to_ref(user, source_sha, branch, target_ref, message, first_parent_ref, allow_conflicts) + def user_merge_to_ref(user, source_sha:, branch:, target_ref:, message:, first_parent_ref:, allow_conflicts: false) request = Gitaly::UserMergeToRefRequest.new( repository: @gitaly_repo, source_sha: source_sha, diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index bd450249355..a93f4071efc 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -339,6 +339,11 @@ module Gitlab search_results_from_response(response, options) end + def search_files_by_regexp(ref, filter) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: '.', filter: filter) + GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) + end + def disconnect_alternates request = Gitaly::DisconnectGitAlternatesRequest.new( repository: @gitaly_repo diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 7edd42f9ef7..dd9e3d5d28b 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -11,7 +11,7 @@ module Gitlab DirectPathAccessError = Class.new(StandardError) InvalidConfigurationError = Class.new(StandardError) - INVALID_STORAGE_MESSAGE = <<~MSG.freeze + INVALID_STORAGE_MESSAGE = <<~MSG Storage is invalid because it has no `path` key. For source installations, update your config/gitlab.yml Refer to gitlab.yml.example for an updated example. diff --git a/lib/gitlab/gitaly_client/wiki_file.rb b/lib/gitlab/gitaly_client/wiki_file.rb deleted file mode 100644 index ef2b23732d1..00000000000 --- a/lib/gitlab/gitaly_client/wiki_file.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GitalyClient - class WikiFile - ATTRS = %i(name mime_type path raw_data).freeze - - include AttributesBag - end - end -end diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 9034edb6263..fecc2b7023d 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -153,32 +153,6 @@ module Gitlab versions end - def find_file(name, revision) - request = Gitaly::WikiFindFileRequest.new( - repository: @gitaly_repo, - name: encode_binary(name), - revision: encode_binary(revision) - ) - - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request, timeout: GitalyClient.fast_timeout) - wiki_file = nil - - response.each do |message| - next unless message.name.present? || wiki_file - - if wiki_file - wiki_file.raw_data = "#{wiki_file.raw_data}#{message.raw_data}" - else - wiki_file = GitalyClient::WikiFile.new(message.to_h) - # All gRPC strings in a response are frozen, so we get - # an unfrozen version here so appending in the else clause below doesn't blow up. - wiki_file.raw_data = wiki_file.raw_data.dup - end - end - - wiki_file - end - private # If a block is given and the yielded value is truthy, iteration will be diff --git a/lib/gitlab/golang.rb b/lib/gitlab/golang.rb index f2dc668c482..31b7a198b92 100644 --- a/lib/gitlab/golang.rb +++ b/lib/gitlab/golang.rb @@ -2,10 +2,12 @@ module Gitlab module Golang + PseudoVersion = Struct.new(:semver, :timestamp, :commit_id) + extend self def local_module_prefix - @gitlab_prefix ||= "#{Settings.build_gitlab_go_url}/".freeze + @gitlab_prefix ||= "#{Settings.build_gitlab_go_url}/" end def semver_tag?(tag) @@ -37,11 +39,11 @@ module Gitlab end # This pattern is intentionally more forgiving than the patterns - # above. Correctness is verified by #pseudo_version_commit. + # above. Correctness is verified by #validate_pseudo_version. /\A\d{14}-\h+\z/.freeze.match? pre end - def pseudo_version_commit(project, semver) + def parse_pseudo_version(semver) # Per Go's implementation of pseudo-versions, a tag should be # considered a pseudo-version if it matches one of the patterns # listed in #pseudo_version?, regardless of the content of the @@ -55,9 +57,14 @@ module Gitlab # - [Pseudo-version request processing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/coderepo.go) # Go ignores anything before '.' or after the second '-', so we will do the same - timestamp, sha = semver.prerelease.split('-').last 2 + timestamp, commit_id = semver.prerelease.split('-').last 2 timestamp = timestamp.split('.').last - commit = project.repository.commit_by(oid: sha) + + PseudoVersion.new(semver, timestamp, commit_id) + end + + def validate_pseudo_version(project, version, commit = nil) + commit ||= project.repository.commit_by(oid: version.commit_id) # Error messages are based on the responses of proxy.golang.org @@ -65,10 +72,10 @@ module Gitlab raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit # Require the SHA fragment to be 12 characters long - raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless sha.length == 12 + raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless version.commit_id.length == 12 # Require the timestamp to match that of the commit - raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == timestamp + raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == version.timestamp commit end @@ -77,6 +84,14 @@ module Gitlab Packages::SemVer.parse(str, prefixed: true) end + def go_path(project, path = nil) + if path.blank? + "#{local_module_prefix}/#{project.full_path}" + else + "#{local_module_prefix}/#{project.full_path}/#{path}" + end + end + def pkg_go_dev_url(name, version = nil) if version "https://pkg.go.dev/#{name}@#{version}" diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index c7e215c143f..08c17058fcb 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -45,7 +45,7 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) - push_frontend_feature_flag(:usage_data_api, default_enabled: true) + push_frontend_feature_flag(:usage_data_api, type: :ops, default_enabled: :yaml) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) end diff --git a/lib/gitlab/grape_logging/loggers/context_logger.rb b/lib/gitlab/grape_logging/loggers/context_logger.rb index 0a8f0872fbe..468a296886e 100644 --- a/lib/gitlab/grape_logging/loggers/context_logger.rb +++ b/lib/gitlab/grape_logging/loggers/context_logger.rb @@ -6,7 +6,7 @@ module Gitlab module Loggers class ContextLogger < ::GrapeLogging::Loggers::Base def parameters(_, _) - Labkit::Context.current.to_h + Gitlab::ApplicationContext.current end end end diff --git a/lib/gitlab/graphql/authorize.rb b/lib/gitlab/graphql/authorize.rb deleted file mode 100644 index e83b567308b..00000000000 --- a/lib/gitlab/graphql/authorize.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - # Allow fields to declare permissions their objects must have. The field - # will be set to nil unless all required permissions are present. - module Authorize - extend ActiveSupport::Concern - - def self.use(schema_definition) - schema_definition.instrument(:field, Gitlab::Graphql::Authorize::Instrumentation.new, after_built_ins: true) - end - end - end -end diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb deleted file mode 100644 index e8db619f88a..00000000000 --- a/lib/gitlab/graphql/authorize/authorize_field_service.rb +++ /dev/null @@ -1,147 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module Authorize - class AuthorizeFieldService - def initialize(field) - @field = field - @old_resolve_proc = @field.resolve_proc - end - - def authorizations? - authorizations.present? - end - - def authorized_resolve - proc do |parent_typed_object, args, ctx| - resolved_type = @old_resolve_proc.call(parent_typed_object, args, ctx) - authorizing_object = authorize_against(parent_typed_object, resolved_type) - - filter_allowed(ctx[:current_user], resolved_type, authorizing_object) - end - end - - private - - def authorizations - @authorizations ||= (type_authorizations + field_authorizations).uniq - end - - # Returns any authorize metadata from the return type of @field - def type_authorizations - type = @field.type - - # When the return type of @field is a collection, find the singular type - if @field.connection? - type = node_type_for_relay_connection(type) - elsif type.list? - type = node_type_for_basic_connection(type) - end - - type = type.unwrap if type.kind.non_null? - - Array.wrap(type.metadata[:authorize]) - end - - # Returns any authorize metadata from @field - def field_authorizations - return [] if @field.metadata[:authorize] == true - - Array.wrap(@field.metadata[:authorize]) - end - - def authorize_against(parent_typed_object, resolved_type) - if scalar_type? - # The field is a built-in/scalar type, or a list of scalars - # authorize using the parent's object - parent_typed_object.object - elsif @field.connection? || @field.type.list? || resolved_type.is_a?(Array) - # The field is a connection or a list of non-built-in types, we'll - # authorize each element when rendering - nil - elsif resolved_type.respond_to?(:object) - # The field is a type representing a single object, we'll authorize - # against the object directly - resolved_type.object - else - # Resolved type is a single object that might not be loaded yet by - # the batchloader, we'll authorize that - resolved_type - end - end - - def filter_allowed(current_user, resolved_type, authorizing_object) - if resolved_type.nil? - # We're not rendering anything, for example when a record was not found - # no need to do anything - elsif authorizing_object - # Authorizing fields representing scalars, or a simple field with an object - ::Gitlab::Graphql::Lazy.with_value(authorizing_object) do |object| - resolved_type if allowed_access?(current_user, object) - end - elsif @field.connection? - ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |type| - # A connection with pagination, modify the visible nodes on the - # connection type in place - nodes = to_nodes(type) - nodes.keep_if { |node| allowed_access?(current_user, node) } if nodes - type - end - elsif @field.type.list? || resolved_type.is_a?(Array) - # A simple list of rendered types each object being an object to authorize - ::Gitlab::Graphql::Lazy.with_value(resolved_type) do |items| - items.select do |single_object_type| - object_type = realized(single_object_type) - object = object_type.try(:object) || object_type - allowed_access?(current_user, object) - end - end - else - raise "Can't authorize #{@field}" - end - end - - # Ensure that we are dealing with realized objects, not delayed promises - def realized(thing) - ::Gitlab::Graphql::Lazy.force(thing) - end - - # Try to get the connection - # can be at type.object or at type - def to_nodes(type) - if type.respond_to?(:nodes) - type.nodes - elsif type.respond_to?(:object) - to_nodes(type.object) - else - nil - end - end - - def allowed_access?(current_user, object) - object = realized(object) - - authorizations.all? do |ability| - Ability.allowed?(current_user, ability, object) - end - end - - # Returns the singular type for relay connections. - # This will be the type class of edges.node - def node_type_for_relay_connection(type) - type.unwrap.get_field('edges').type.unwrap.get_field('node').type - end - - # Returns the singular type for basic connections, for example `[Types::ProjectType]` - def node_type_for_basic_connection(type) - type.unwrap - end - - def scalar_type? - node_type_for_basic_connection(@field.type).kind.scalar? - end - end - end - end -end diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index 6ee446011d4..4d575b964e5 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -5,15 +5,17 @@ module Gitlab module Authorize module AuthorizeResource extend ActiveSupport::Concern + ConfigurationError = Class.new(StandardError) - RESOURCE_ACCESS_ERROR = "The resource that you are attempting to access does not exist or you don't have permission to perform this action" + RESOURCE_ACCESS_ERROR = "The resource that you are attempting to access does " \ + "not exist or you don't have permission to perform this action" class_methods do def required_permissions # If the `#authorize` call is used on multiple classes, we add the # permissions specified on a subclass, to the ones that were specified - # on it's superclass. - @required_permissions ||= if self.respond_to?(:superclass) && superclass.respond_to?(:required_permissions) + # on its superclass. + @required_permissions ||= if respond_to?(:superclass) && superclass.respond_to?(:required_permissions) superclass.required_permissions.dup else [] @@ -23,6 +25,18 @@ module Gitlab def authorize(*permissions) required_permissions.concat(permissions) end + + def authorizes_object? + defined?(@authorizes_object) ? @authorizes_object : false + end + + def authorizes_object! + @authorizes_object = true + end + + def raise_resource_not_available_error!(msg = RESOURCE_ACCESS_ERROR) + raise ::Gitlab::Graphql::Errors::ResourceNotAvailable, msg + end end def find_object(*args) @@ -37,33 +51,21 @@ module Gitlab object end + # authorizes the object using the current class authorization. def authorize!(object) - unless authorized_resource?(object) - raise_resource_not_available_error! - end + raise_resource_not_available_error! unless authorized_resource?(object) end - # this was named `#authorized?`, however it conflicts with the native - # graphql gem version - # TODO consider adopting the gem's built in authorization system - # https://gitlab.com/gitlab-org/gitlab/issues/13984 def authorized_resource?(object) # Sanity check. We don't want to accidentally allow a developer to authorize # without first adding permissions to authorize against - if self.class.required_permissions.empty? - raise Gitlab::Graphql::Errors::ArgumentError, "#{self.class.name} has no authorizations" - end + raise ConfigurationError, "#{self.class.name} has no authorizations" if self.class.authorization.none? - self.class.required_permissions.all? do |ability| - # The actions could be performed across multiple objects. In which - # case the current user is common, and we could benefit from the - # caching in `DeclarativePolicy`. - Ability.allowed?(current_user, ability, object, scope: :user) - end + self.class.authorization.ok?(object, current_user) end - def raise_resource_not_available_error!(msg = RESOURCE_ACCESS_ERROR) - raise Gitlab::Graphql::Errors::ResourceNotAvailable, msg + def raise_resource_not_available_error!(*args) + self.class.raise_resource_not_available_error!(*args) end end end diff --git a/lib/gitlab/graphql/authorize/connection_filter_extension.rb b/lib/gitlab/graphql/authorize/connection_filter_extension.rb new file mode 100644 index 00000000000..c75510df3e3 --- /dev/null +++ b/lib/gitlab/graphql/authorize/connection_filter_extension.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Authorize + class ConnectionFilterExtension < GraphQL::Schema::FieldExtension + class Redactor + include ::Gitlab::Graphql::Laziness + + def initialize(type, context) + @type = type + @context = context + end + + def redact(nodes) + remove_unauthorized(nodes) + + nodes + end + + def active? + # some scalar types (such as integers) do not respond to :authorized? + return false unless @type.respond_to?(:authorized?) + + auth = @type.try(:authorization) + + auth.nil? || auth.any? + end + + private + + def remove_unauthorized(nodes) + nodes + .map! { |lazy| force(lazy) } + .keep_if { |forced| @type.authorized?(forced, @context) } + end + end + + def after_resolve(value:, context:, **rest) + return value if value.is_a?(GraphQL::Execution::Execute::Skip) + + if @field.connection? + redact_connection(value, context) + elsif @field.type.list? + redact_list(value.to_a, context) unless value.nil? + end + + value + end + + def redact_connection(conn, context) + redactor = Redactor.new(@field.type.unwrap.node_type, context) + return unless redactor.active? + + conn.redactor = redactor if conn.respond_to?(:redactor=) + end + + def redact_list(list, context) + redactor = Redactor.new(@field.type.unwrap, context) + redactor.redact(list) if redactor.active? + end + end + end + end +end diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb deleted file mode 100644 index 15ecc3b04f0..00000000000 --- a/lib/gitlab/graphql/authorize/instrumentation.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module Authorize - class Instrumentation - # Replace the resolver for the field with one that will only return the - # resolved object if the permissions check is successful. - def instrument(_type, field) - service = AuthorizeFieldService.new(field) - - if service.authorizations? - field.redefine { resolve(service.authorized_resolve) } - else - field - end - end - end - end - end -end diff --git a/lib/gitlab/graphql/authorize/object_authorization.rb b/lib/gitlab/graphql/authorize/object_authorization.rb new file mode 100644 index 00000000000..0bc87108871 --- /dev/null +++ b/lib/gitlab/graphql/authorize/object_authorization.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Authorize + class ObjectAuthorization + attr_reader :abilities + + def initialize(abilities) + @abilities = Array.wrap(abilities).flatten + end + + def none? + abilities.empty? + end + + def any? + abilities.present? + end + + def ok?(object, current_user) + return true if none? + + subject = object.try(:declarative_policy_subject) || object + abilities.all? do |ability| + Ability.allowed?(current_user, ability, subject) + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/deprecation.rb b/lib/gitlab/graphql/deprecation.rb new file mode 100644 index 00000000000..e0176e2d6e0 --- /dev/null +++ b/lib/gitlab/graphql/deprecation.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + class Deprecation + REASONS = { + renamed: 'This was renamed.', + discouraged: 'Use of this is not recommended.' + }.freeze + + include ActiveModel::Validations + + validates :milestone, presence: true, format: { with: /\A\d+\.\d+\z/, message: 'must be milestone-ish' } + validates :reason, presence: true + validates :reason, + format: { with: /.*[^.]\z/, message: 'must not end with a period' }, + if: :reason_is_string? + validate :milestone_is_string + validate :reason_known_or_string + + def self.parse(options) + new(**options) if options + end + + def initialize(reason: nil, milestone: nil, replacement: nil) + @reason = reason.presence + @milestone = milestone.presence + @replacement = replacement.presence + end + + def ==(other) + return false unless other.is_a?(self.class) + + [reason_text, milestone, replacement] == [:reason_text, :milestone, :replacement].map do |attr| + other.send(attr) # rubocop: disable GitlabSecurity/PublicSend + end + end + alias_method :eql, :== + + def markdown(context: :inline) + parts = [ + "#{deprecated_in(format: :markdown)}.", + reason_text, + replacement.then { |r| "Use: `#{r}`." if r } + ].compact + + case context + when :block + ['WARNING:', *parts].join("\n") + when :inline + parts.join(' ') + end + end + + def edit_description(original_description) + @original_description = original_description + return unless original_description + + original_description + description_suffix + end + + def original_description + return unless @original_description + return @original_description if @original_description.ends_with?('.') + + "#{@original_description}." + end + + def deprecation_reason + [ + reason_text, + replacement && "Please use `#{replacement}`.", + "#{deprecated_in}." + ].compact.join(' ') + end + + private + + attr_reader :reason, :milestone, :replacement + + def milestone_is_string + return if milestone.is_a?(String) + + errors.add(:milestone, 'must be a string') + end + + def reason_known_or_string + return if REASONS.key?(reason) + return if reason_is_string? + + errors.add(:reason, 'must be a known reason or a string') + end + + def reason_is_string? + reason.is_a?(String) + end + + def reason_text + @reason_text ||= REASONS[reason] || "#{reason.to_s.strip}." + end + + def description_suffix + " #{deprecated_in}: #{reason_text}" + end + + def deprecated_in(format: :plain) + case format + when :plain + "Deprecated in #{milestone}" + when :markdown + "**Deprecated** in #{milestone}" + end + end + end + end +end diff --git a/lib/gitlab/graphql/docs/helper.rb b/lib/gitlab/graphql/docs/helper.rb index e9ff85d9ca9..f4173e26224 100644 --- a/lib/gitlab/graphql/docs/helper.rb +++ b/lib/gitlab/graphql/docs/helper.rb @@ -27,7 +27,10 @@ module Gitlab MD end - def render_name_and_description(object, level = 3) + # Template methods: + # Methods that return chunks of Markdown for insertion into the document + + def render_name_and_description(object, owner: nil, level: 3) content = [] content << "#{'#' * level} `#{object[:name]}`" @@ -35,10 +38,22 @@ module Gitlab if object[:description].present? desc = object[:description].strip desc += '.' unless desc.ends_with?('.') + end + + if object[:is_deprecated] + owner = Array.wrap(owner) + deprecation = schema_deprecation(owner, object[:name]) + content << (deprecation&.original_description || desc) + content << render_deprecation(object, owner, :block) + else content << desc end - content.join("\n\n") + content.compact.join("\n\n") + end + + def render_return_type(query) + "Returns #{render_field_type(query[:type])}.\n" end def sorted_by_name(objects) @@ -47,39 +62,25 @@ module Gitlab objects.sort_by { |o| o[:name] } end - def render_field(field) - row(render_name(field), render_field_type(field[:type]), render_description(field)) + def render_field(field, owner) + render_row( + render_name(field, owner), + render_field_type(field[:type]), + render_description(field, owner, :inline) + ) end - def render_enum_value(value) - row(render_name(value), render_description(value)) + def render_enum_value(enum, value) + render_row(render_name(value, enum[:name]), render_description(value, enum[:name], :inline)) end - def row(*values) - "| #{values.join(' | ')} |" + def render_union_member(member) + "- [`#{member}`](##{member.downcase})" end - def render_name(object) - rendered_name = "`#{object[:name]}`" - rendered_name += ' **{warning-solid}**' if object[:is_deprecated] - rendered_name - end + # QUERIES: - # Returns the object description. If the object has been deprecated, - # the deprecation reason will be returned in place of the description. - def render_description(object) - return object[:description] unless object[:is_deprecated] - - "**Deprecated:** #{object[:deprecation_reason]}" - end - - def render_field_type(type) - "[`#{type[:info]}`](##{type[:name].downcase})" - end - - def render_return_type(query) - "Returns #{render_field_type(query[:type])}.\n" - end + # Methods that return parts of the schema, or related information: # We are ignoring connections and built in types for now, # they should be added when queries are generated. @@ -103,6 +104,83 @@ module Gitlab !enum_type[:name].in?(%w[__DirectiveLocation __TypeKind]) end end + + private # DO NOT CALL THESE METHODS IN TEMPLATES + + # Template methods + + def render_row(*values) + "| #{values.map { |val| val.to_s.squish }.join(' | ')} |" + end + + def render_name(object, owner = nil) + rendered_name = "`#{object[:name]}`" + rendered_name += ' **{warning-solid}**' if object[:is_deprecated] + rendered_name + end + + # Returns the object description. If the object has been deprecated, + # the deprecation reason will be returned in place of the description. + def render_description(object, owner = nil, context = :block) + owner = Array.wrap(owner) + return render_deprecation(object, owner, context) if object[:is_deprecated] + return if object[:description].blank? + + desc = object[:description].strip + desc += '.' unless desc.ends_with?('.') + desc + end + + def render_deprecation(object, owner, context) + deprecation = schema_deprecation(owner, object[:name]) + return deprecation.markdown(context: context) if deprecation + + reason = object[:deprecation_reason] || 'Use of this is deprecated.' + "**Deprecated:** #{reason}" + end + + def render_field_type(type) + "[`#{type[:info]}`](##{type[:name].downcase})" + end + + # Queries + + # returns the deprecation information for a field or argument + # See: Gitlab::Graphql::Deprecation + def schema_deprecation(type_name, field_name) + schema_member(type_name, field_name)&.deprecation + end + + # Return a part of the schema. + # + # This queries the Schema by owner and name to find: + # + # - fields (e.g. `schema_member('Query', 'currentUser')`) + # - arguments (e.g. `schema_member(['Query', 'project], 'fullPath')`) + def schema_member(type_name, field_name) + type_name = Array.wrap(type_name) + if type_name.size == 2 + arg_name = field_name + type_name, field_name = type_name + else + type_name = type_name.first + arg_name = nil + end + + return if type_name.nil? || field_name.nil? + + type = schema.types[type_name] + return unless type && type.kind.fields? + + field = type.fields[field_name] + return field if arg_name.nil? + + args = field.arguments + is_mutation = field.mutation && field.mutation <= ::Mutations::BaseMutation + args = args['input'].type.unwrap.arguments if is_mutation + + args[arg_name] + end end end end diff --git a/lib/gitlab/graphql/docs/renderer.rb b/lib/gitlab/graphql/docs/renderer.rb index 6abd56c89c6..497567f9389 100644 --- a/lib/gitlab/graphql/docs/renderer.rb +++ b/lib/gitlab/graphql/docs/renderer.rb @@ -10,17 +10,20 @@ module Gitlab # It uses graphql-docs helpers and schema parser, more information in https://github.com/gjtorikian/graphql-docs. # # Arguments: - # schema - the GraphQL schema definition. For GitLab should be: GitlabSchema.graphql_definition + # schema - the GraphQL schema definition. For GitLab should be: GitlabSchema # output_dir: The folder where the markdown files will be saved # template: The path of the haml template to be parsed class Renderer include Gitlab::Graphql::Docs::Helper + attr_reader :schema + def initialize(schema, output_dir:, template:) @output_dir = output_dir @template = template @layout = Haml::Engine.new(File.read(template)) - @parsed_schema = GraphQLDocs::Parser.new(schema, {}).parse + @parsed_schema = GraphQLDocs::Parser.new(schema.graphql_definition, {}).parse + @schema = schema end def contents diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index 847f1777b08..fe73297d0d9 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -27,7 +27,7 @@ \ - sorted_by_name(queries).each do |query| - = render_name_and_description(query) + = render_name_and_description(query, owner: 'Query') \ = render_return_type(query) - unless query[:arguments].empty? @@ -35,7 +35,7 @@ ~ "| Name | Type | Description |" ~ "| ---- | ---- | ----------- |" - sorted_by_name(query[:arguments]).each do |argument| - = render_field(argument) + = render_field(argument, query[:type][:name]) \ :plain @@ -58,7 +58,7 @@ ~ "| Field | Type | Description |" ~ "| ----- | ---- | ----------- |" - sorted_by_name(type[:fields]).each do |field| - = render_field(field) + = render_field(field, type[:name]) \ :plain @@ -79,7 +79,7 @@ ~ "| Value | Description |" ~ "| ----- | ----------- |" - sorted_by_name(enum[:values]).each do |value| - = render_enum_value(value) + = render_enum_value(enum, value) \ :plain @@ -121,12 +121,12 @@ \ - graphql_union_types.each do |type| - = render_name_and_description(type, 4) + = render_name_and_description(type, level: 4) \ One of: \ - - type[:possible_types].each do |type_name| - ~ "- [`#{type_name}`](##{type_name.downcase})" + - type[:possible_types].each do |member| + = render_union_member(member) \ :plain @@ -134,7 +134,7 @@ \ - graphql_interface_types.each do |type| - = render_name_and_description(type, 4) + = render_name_and_description(type, level: 4) \ Implementations: \ @@ -144,5 +144,5 @@ ~ "| Field | Type | Description |" ~ "| ----- | ---- | ----------- |" - sorted_by_name(type[:fields] + type[:connections]).each do |field| - = render_field(field) + = render_field(field, type[:name]) \ diff --git a/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb b/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb index 67511c124e4..1945388cdd4 100644 --- a/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb +++ b/lib/gitlab/graphql/loaders/batch_lfs_oid_loader.rb @@ -5,7 +5,8 @@ module Gitlab module Loaders class BatchLfsOidLoader def initialize(repository, blob_id) - @repository, @blob_id = repository, blob_id + @repository = repository + @blob_id = blob_id end def find diff --git a/lib/gitlab/graphql/loaders/batch_model_loader.rb b/lib/gitlab/graphql/loaders/batch_model_loader.rb index 9b85ba164d4..805864cdd4c 100644 --- a/lib/gitlab/graphql/loaders/batch_model_loader.rb +++ b/lib/gitlab/graphql/loaders/batch_model_loader.rb @@ -7,7 +7,8 @@ module Gitlab attr_reader :model_class, :model_id def initialize(model_class, model_id) - @model_class, @model_id = model_class, model_id + @model_class = model_class + @model_id = model_id end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/graphql/loaders/full_path_model_loader.rb b/lib/gitlab/graphql/loaders/full_path_model_loader.rb index 0aa237c78de..26c1ce64a83 100644 --- a/lib/gitlab/graphql/loaders/full_path_model_loader.rb +++ b/lib/gitlab/graphql/loaders/full_path_model_loader.rb @@ -9,7 +9,8 @@ module Gitlab attr_reader :model_class, :full_path def initialize(model_class, full_path) - @model_class, @full_path = model_class, full_path + @model_class = model_class + @full_path = full_path end def find diff --git a/lib/gitlab/graphql/negatable_arguments.rb b/lib/gitlab/graphql/negatable_arguments.rb new file mode 100644 index 00000000000..b4ab31ed51a --- /dev/null +++ b/lib/gitlab/graphql/negatable_arguments.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module NegatableArguments + class TypeDefiner + def initialize(resolver_class, type_definition) + @resolver_class = resolver_class + @type_definition = type_definition + end + + def define! + negated_params_type.instance_eval(&@type_definition) + end + + def negated_params_type + @negated_params_type ||= existing_type || build_type + end + + private + + def existing_type + ::Types.const_get(type_class_name, false) if ::Types.const_defined?(type_class_name) + end + + def build_type + klass = Class.new(::Types::BaseInputObject) + ::Types.const_set(type_class_name, klass) + klass + end + + def type_class_name + @type_class_name ||= begin + base_name = @resolver_class.name.sub('Resolvers::', '') + base_name + 'NegatedParamsType' + end + end + end + + def negated(param_key: :not, &block) + definer = ::Gitlab::Graphql::NegatableArguments::TypeDefiner.new(self, block) + definer.define! + + argument param_key, definer.negated_params_type, + required: false, + description: <<~MD + List of negated arguments. + Warning: this argument is experimental and a subject to change in future. + MD + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb index bd785880b57..6645dac36fa 100644 --- a/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb +++ b/lib/gitlab/graphql/pagination/keyset/conditions/base_condition.rb @@ -13,7 +13,11 @@ module Gitlab # @param [Symbol] before_or_after indicates whether we want # items :before the cursor or :after the cursor def initialize(arel_table, order_list, values, operators, before_or_after) - @arel_table, @order_list, @values, @operators, @before_or_after = arel_table, order_list, values, operators, before_or_after + @arel_table = arel_table + @order_list = order_list + @values = values + @operators = operators + @before_or_after = before_or_after @before_or_after = :after unless [:after, :before].include?(@before_or_after) end diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb index 3164598b7b9..ec70f5c5a24 100644 --- a/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb +++ b/lib/gitlab/graphql/pagination/keyset/conditions/not_null_condition.rb @@ -30,15 +30,13 @@ module Gitlab # ex: " OR (relative_position = 23 AND id > 500)" def second_attribute_condition - condition = <<~SQL + <<~SQL OR ( #{table_condition(order_list.first, values.first, '=').to_sql} AND #{table_condition(order_list[1], values[1], operators[1]).to_sql} ) SQL - - condition end # ex: " OR (relative_position IS NULL)" diff --git a/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb b/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb index fa25181d663..1aae1020e79 100644 --- a/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb +++ b/lib/gitlab/graphql/pagination/keyset/conditions/null_condition.rb @@ -14,15 +14,13 @@ module Gitlab # ex: "(relative_position IS NULL AND id > 500)" def first_attribute_condition - condition = <<~SQL + <<~SQL ( #{table_condition(order_list.first, nil, 'is_null').to_sql} AND #{table_condition(order_list[1], values[1], operators[1]).to_sql} ) SQL - - condition end # ex: " OR (relative_position IS NOT NULL)" diff --git a/lib/gitlab/graphql/pagination/keyset/query_builder.rb b/lib/gitlab/graphql/pagination/keyset/query_builder.rb index 29169449843..ee9c902c735 100644 --- a/lib/gitlab/graphql/pagination/keyset/query_builder.rb +++ b/lib/gitlab/graphql/pagination/keyset/query_builder.rb @@ -6,7 +6,10 @@ module Gitlab module Keyset class QueryBuilder def initialize(arel_table, order_list, decoded_cursor, before_or_after) - @arel_table, @order_list, @decoded_cursor, @before_or_after = arel_table, order_list, decoded_cursor, before_or_after + @arel_table = arel_table + @order_list = order_list + @decoded_cursor = decoded_cursor + @before_or_after = before_or_after if order_list.empty? raise ArgumentError.new('No ordering scopes have been supplied') diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb index fcf293fb13e..74f55abccbc 100644 --- a/lib/gitlab/graphql/queries.rb +++ b/lib/gitlab/graphql/queries.rb @@ -224,11 +224,9 @@ module Gitlab frag_path = frag_path.gsub(DOTS_RE) do |dots| rel_dir(dots.split('/').count) end - frag_path = frag_path.gsub(IMPLICIT_ROOT) do + frag_path.gsub(IMPLICIT_ROOT) do (Rails.root / 'app').to_s + '/' end - - frag_path end def rel_dir(n_steps_up) diff --git a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb index 8acd27869a9..c6f22e0bd4f 100644 --- a/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb +++ b/lib/gitlab/graphql/query_analyzers/logger_analyzer.rb @@ -12,6 +12,7 @@ module Gitlab def initial_value(query) variables = process_variables(query.provided_variables) default_initial_values(query).merge({ + operation_name: query.operation_name, query_string: query.query_string, variables: variables }) @@ -20,8 +21,8 @@ module Gitlab default_initial_values(query) end - def call(memo, visit_type, irep_node) - RequestStore.store[:graphql_logs] = memo + def call(memo, *) + memo end def final_value(memo) @@ -37,6 +38,8 @@ module Gitlab memo[:used_fields] = field_usages.first memo[:used_deprecated_fields] = field_usages.second + RequestStore.store[:graphql_logs] ||= [] + RequestStore.store[:graphql_logs] << memo GraphqlLogger.info(memo.except!(:time_started, :query)) rescue => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb index e780bf8a986..f5f142c251f 100644 --- a/lib/gitlab/health_checks/gitaly_check.rb +++ b/lib/gitlab/health_checks/gitaly_check.rb @@ -5,7 +5,7 @@ module Gitlab class GitalyCheck extend BaseAbstractCheck - METRIC_PREFIX = 'gitaly_health_check'.freeze + METRIC_PREFIX = 'gitaly_health_check' class << self def readiness diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index 40dee0142b9..765d3dfca56 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -20,7 +20,9 @@ module Gitlab @blob_content = blob_content end - def highlight(text, continue: true, plain: false) + def highlight(text, continue: false, plain: false, context: {}) + @context = context + plain ||= text.length > MAXIMUM_TEXT_HIGHLIGHT_SIZE highlighted_text = highlight_text(text, continue: continue, plain: plain) @@ -31,13 +33,15 @@ module Gitlab def lexer @lexer ||= custom_language || begin Rouge::Lexer.guess(filename: @blob_name, source: @blob_content).new - rescue Rouge::Guesser::Ambiguous => e - e.alternatives.min_by(&:tag) + rescue Rouge::Guesser::Ambiguous => e + e.alternatives.min_by(&:tag) end end private + attr_reader :context + def custom_language return unless @language @@ -53,13 +57,13 @@ module Gitlab end def highlight_plain(text) - @formatter.format(Rouge::Lexers::PlainText.lex(text)).html_safe + @formatter.format(Rouge::Lexers::PlainText.lex(text), context).html_safe end def highlight_rich(text, continue: true) tag = lexer.tag tokens = lexer.lex(text, continue: continue) - Timeout.timeout(timeout_time) { @formatter.format(tokens, tag: tag).html_safe } + Timeout.timeout(timeout_time) { @formatter.format(tokens, context.merge(tag: tag)).html_safe } rescue Timeout::Error => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) highlight_plain(text) diff --git a/lib/gitlab/hook_data/user_builder.rb b/lib/gitlab/hook_data/user_builder.rb new file mode 100644 index 00000000000..537245e948f --- /dev/null +++ b/lib/gitlab/hook_data/user_builder.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class UserBuilder < BaseBuilder + alias_method :user, :object + + # Sample data + # { + # :created_at=>"2021-04-02T10:00:26Z", + # :updated_at=>"2021-04-02T10:00:26Z", + # :event_name=>"user_create", + # :name=>"John Doe", + # :email=>"john@example.com", + # :user_id=>1, + # :username=>"johndoe" + # } + + def build(event) + [ + timestamps_data, + event_data(event), + user_data, + event_specific_user_data(event) + ].reduce(:merge) + end + + private + + def user_data + { + name: user.name, + email: user.email, + user_id: user.id, + username: user.username + } + end + + def event_specific_user_data(event) + case event + when :rename + { old_username: user.username_before_last_save } + when :failed_login + { state: user.state } + else + {} + end + end + end + end +end + +Gitlab::HookData::UserBuilder.prepend_if_ee('EE::Gitlab::HookData::UserBuilder') diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index 37f618ae879..f7a3da53fdb 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -17,14 +17,6 @@ module Gitlab def connection @uri, hostname = validate_url!(uri) - if options.key?(:http_proxyaddr) - proxy_uri_with_port = uri_with_port(options[:http_proxyaddr], options[:http_proxyport]) - proxy_uri_validated = validate_url!(proxy_uri_with_port).first - - @options[:http_proxyaddr] = proxy_uri_validated.omit(:port).to_s - @options[:http_proxyport] = proxy_uri_validated.port - end - super.tap do |http| http.hostname_override = hostname if hostname end @@ -53,11 +45,5 @@ module Gitlab def allow_settings_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end - - def uri_with_port(address, port) - uri = Addressable::URI.parse(address) - uri.port = port if port.present? - uri - end end end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index d60bc79df4c..05a4a8f4c93 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -6,7 +6,7 @@ module Gitlab class RelationFactory include Gitlab::Utils::StrongMemoize - IMPORTED_OBJECT_MAX_RETRIES = 5.freeze + IMPORTED_OBJECT_MAX_RETRIES = 5 OVERRIDES = {}.freeze EXISTING_OBJECT_RELATIONS = %i[].freeze diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 778b42f4358..42d32593cbd 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -265,6 +265,7 @@ excluded_attributes: - :issue_id push_event_payload: - :event_id + - :event_id_convert_to_bigint project_badges: - :group_id resource_label_events: @@ -287,6 +288,7 @@ excluded_attributes: - :label_id events: - :target_id + - :id_convert_to_bigint timelogs: - :issue_id - :merge_request_id diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 428bcbe8dc5..2f15cdd7506 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -76,7 +76,7 @@ module Gitlab def project_uploads_except_avatar(avatar_path) return @project.uploads unless avatar_path - @project.uploads.where("path != ?", avatar_path) + @project.uploads.where.not(path: avatar_path) end def download_and_copy(upload) diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 88753e80391..95c002edf0a 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -28,7 +28,7 @@ module Gitlab prepend_if_ee('EE::Gitlab::ImportSources') # rubocop: disable Cop/InjectEnterpriseEditionModule def options - Hash[import_table.map { |importer| [importer.title, importer.name] }] + import_table.to_h { |importer| [importer.title, importer.name] } end def values diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 61de6b02453..a865a6392f0 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -6,24 +6,6 @@ module Gitlab DURATION_PRECISION = 6 # microseconds - def keys - @keys ||= [ - :cpu_s, - :gitaly_calls, - :gitaly_duration_s, - :rugged_calls, - :rugged_duration_s, - :elasticsearch_calls, - :elasticsearch_duration_s, - :elasticsearch_timed_out_count, - *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values, - *::Gitlab::Instrumentation::Redis.known_payload_keys, - *::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS, - *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS, - *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS - ] - end - def init_instrumentation_data(request_ip: nil) # Set `request_start_time` only if this is request # This is done, as `request_start_time` imply `request_deadline` diff --git a/lib/gitlab/issuables_count_for_state.rb b/lib/gitlab/issuables_count_for_state.rb index 945ab7f40c2..6b33b60e850 100644 --- a/lib/gitlab/issuables_count_for_state.rb +++ b/lib/gitlab/issuables_count_for_state.rb @@ -78,7 +78,7 @@ module Gitlab # to perform the calculation more efficiently. Until then, use a shorter # timeout and return -1 as a sentinel value if it is triggered begin - ApplicationRecord.with_fast_statement_timeout do + ApplicationRecord.with_fast_read_statement_timeout do finder.count_by_state end rescue ActiveRecord::QueryCanceled => err diff --git a/lib/gitlab/jira/dvcs.rb b/lib/gitlab/jira/dvcs.rb index 4415f98fc7f..ddf2cd76709 100644 --- a/lib/gitlab/jira/dvcs.rb +++ b/lib/gitlab/jira/dvcs.rb @@ -3,8 +3,8 @@ module Gitlab module Jira module Dvcs - ENCODED_SLASH = '@'.freeze - SLASH = '/'.freeze + ENCODED_SLASH = '@' + SLASH = '/' ENCODED_ROUTE_REGEX = /[a-zA-Z0-9_\-\.#{ENCODED_SLASH}]+/.freeze def self.encode_slash(path) diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 8565f664cd4..b51c0a33457 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -186,9 +186,14 @@ module Gitlab # The `env` param is ignored because it's not needed in either our formatter or Grape's, # but it is passed through for consistency. # + # If explicitly supplied with a `PrecompiledJson` instance it will skip conversion + # and return it directly. This is mostly used in caching. + # # @param object [Object] # @return [String] def self.call(object, env = nil) + return object.to_s if object.is_a?(PrecompiledJson) + if Feature.enabled?(:grape_gitlab_json, default_enabled: true) Gitlab::Json.dump(object) else @@ -197,6 +202,34 @@ module Gitlab end end + # Wrapper class used to skip JSON dumping on Grape endpoints. + + class PrecompiledJson + UnsupportedFormatError = Class.new(StandardError) + + # @overload PrecompiledJson.new("foo") + # @param value [String] + # + # @overload PrecompiledJson.new(["foo", "bar"]) + # @param value [Array<String>] + def initialize(value) + @value = value + end + + # Convert the value to a String. This will invoke + # `#to_s` on the members of the value if it's an array. + # + # @return [String] + # @raise [NoMethodError] if the objects in an array doesn't support to_s + # @raise [PrecompiledJson::UnsupportedFormatError] if the value is neither a String or Array + def to_s + return @value if @value.is_a?(String) + return "[#{@value.join(',')}]" if @value.is_a?(Array) + + raise UnsupportedFormatError + end + end + class LimitedEncoder LimitExceeded = Class.new(StandardError) diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 329c0f221b5..7a674cb5c21 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -27,7 +27,7 @@ module Gitlab def included_in_gitlab_com_rollout?(project) return true unless ::Gitlab.com? - Feature.enabled?(:kubernetes_agent_on_gitlab_com, project) + Feature.enabled?(:kubernetes_agent_on_gitlab_com, project, default_enabled: :yaml) end end end diff --git a/lib/gitlab/kubernetes/deployment.rb b/lib/gitlab/kubernetes/deployment.rb index 55ed9a7517e..f2e3a0e6810 100644 --- a/lib/gitlab/kubernetes/deployment.rb +++ b/lib/gitlab/kubernetes/deployment.rb @@ -5,7 +5,7 @@ module Gitlab class Deployment include Gitlab::Utils::StrongMemoize - STABLE_TRACK_VALUE = 'stable'.freeze + STABLE_TRACK_VALUE = 'stable' def initialize(attributes = {}, pods: []) @attributes = attributes diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb index 7600e60b904..1e5edb79f10 100644 --- a/lib/gitlab/language_detection.rb +++ b/lib/gitlab/language_detection.rb @@ -20,7 +20,7 @@ module Gitlab # Newly detected languages, returned in a structure accepted by # Gitlab::Database.bulk_insert def insertions(programming_languages) - lang_to_id = programming_languages.map { |p| [p.name, p.id] }.to_h + lang_to_id = programming_languages.to_h { |p| [p.name, p.id] } (languages - previous_language_names).map do |new_lang| { @@ -63,8 +63,7 @@ module Gitlab @repository .languages .first(MAX_LANGUAGES) - .map { |l| [l[:label], l] } - .to_h + .to_h { |l| [l[:label], l] } end end end diff --git a/lib/gitlab/manifest_import/manifest.rb b/lib/gitlab/manifest_import/manifest.rb index 7208fe5bbc5..618ddf37b88 100644 --- a/lib/gitlab/manifest_import/manifest.rb +++ b/lib/gitlab/manifest_import/manifest.rb @@ -47,6 +47,10 @@ module Gitlab @errors << 'Make sure every <project> tag has name and path attributes.' end + unless validate_scheme + @errors << 'Make sure the url does not start with javascript' + end + @errors.empty? end @@ -64,6 +68,10 @@ module Gitlab end end + def validate_scheme + remote !~ /\Ajavascript/i + end + def repository_url(name) Gitlab::Utils.append_path(remote, name) end diff --git a/lib/gitlab/marker_range.rb b/lib/gitlab/marker_range.rb index 50a59adebdf..73e4a545679 100644 --- a/lib/gitlab/marker_range.rb +++ b/lib/gitlab/marker_range.rb @@ -24,6 +24,12 @@ module Gitlab Range.new(self.begin, self.end, self.exclude_end?) end + def ==(other) + return false unless other.is_a?(self.class) + + self.mode == other.mode && super + end + attr_reader :mode end end diff --git a/lib/gitlab/markup_helper.rb b/lib/gitlab/markup_helper.rb index d419fa66e57..45c6205b36b 100644 --- a/lib/gitlab/markup_helper.rb +++ b/lib/gitlab/markup_helper.rb @@ -4,7 +4,7 @@ module Gitlab module MarkupHelper extend self - MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown].freeze + MARKDOWN_EXTENSIONS = %w[mdown mkd mkdn md markdown rmd].freeze ASCIIDOC_EXTENSIONS = %w[adoc ad asciidoc].freeze OTHER_EXTENSIONS = %w[textile rdoc org creole wiki mediawiki rst].freeze EXTENSIONS = MARKDOWN_EXTENSIONS + ASCIIDOC_EXTENSIONS + OTHER_EXTENSIONS diff --git a/lib/gitlab/metrics/background_transaction.rb b/lib/gitlab/metrics/background_transaction.rb index 3dda68bf93f..a1fabe75a97 100644 --- a/lib/gitlab/metrics/background_transaction.rb +++ b/lib/gitlab/metrics/background_transaction.rb @@ -34,8 +34,9 @@ module Gitlab def labels @labels ||= { - endpoint_id: current_context&.get_attribute(:caller_id), - feature_category: current_context&.get_attribute(:feature_category) + endpoint_id: endpoint_id, + feature_category: feature_category, + queue: queue } end @@ -44,6 +45,21 @@ module Gitlab def current_context Labkit::Context.current end + + def feature_category + current_context&.get_attribute(:feature_category) + end + + def endpoint_id + current_context&.get_attribute(:caller_id) + end + + def queue + worker_class = endpoint_id.to_s.safe_constantize + return if worker_class.blank? || !worker_class.respond_to?(:queue) + + worker_class.queue.to_s + end end end end diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb index c90c1e3f0bc..55d14d6f94a 100644 --- a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb +++ b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb @@ -104,9 +104,7 @@ module Gitlab def format_query(metric) expression = remove_new_lines(metric[:expr]) expression = replace_variables(expression) - expression = replace_global_variables(expression, metric) - - expression + replace_global_variables(expression, metric) end # Accomodates instance-defined Grafana variables. @@ -135,9 +133,7 @@ module Gitlab def replace_global_variables(expression, metric) expression = expression.gsub('$__interval', metric[:interval]) if metric[:interval] expression = expression.gsub('$__from', query_params[:from]) - expression = expression.gsub('$__to', query_params[:to]) - - expression + expression.gsub('$__to', query_params[:to]) end # Removes new lines from expression. diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index 60ae22df607..c0336a4d0fb 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -32,9 +32,9 @@ module Gitlab private def init_metrics - METRIC_DESCRIPTIONS.map do |name, description| + METRIC_DESCRIPTIONS.to_h do |name, description| [name, ::Gitlab::Metrics.gauge(:"#{METRIC_PREFIX}#{name}", description)] - end.to_h + end end def host_stats diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 5eefef02507..0d1cd641ffe 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -11,13 +11,16 @@ module Gitlab DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze SQL_COMMANDS_WITH_COMMENTS_REGEX = /\A(\/\*.*\*\/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i.freeze - DURATION_BUCKET = [0.05, 0.1, 0.25].freeze + SQL_DURATION_BUCKET = [0.05, 0.1, 0.25].freeze + TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze # This event is published from ActiveRecordBaseTransactionMetrics and # used to record a database transaction duration when calling # ActiveRecord::Base.transaction {} block. def transaction(event) - observe(:gitlab_database_transaction_seconds, event) + observe(:gitlab_database_transaction_seconds, event) do + buckets TRANSACTION_DURATION_BUCKET + end end def sql(event) @@ -33,7 +36,9 @@ module Gitlab increment(:db_cached_count) if cached_query?(payload) increment(:db_write_count) unless select_sql_command?(payload) - observe(:gitlab_sql_duration_seconds, event) + observe(:gitlab_sql_duration_seconds, event) do + buckets SQL_DURATION_BUCKET + end end def self.db_counter_payload @@ -46,6 +51,10 @@ module Gitlab payload end + def self.known_payload_keys + DB_COUNTERS + end + private def ignored_query?(payload) @@ -66,10 +75,8 @@ module Gitlab Gitlab::SafeRequestStore[counter] = Gitlab::SafeRequestStore[counter].to_i + 1 end - def observe(histogram, event) - current_transaction&.observe(histogram, event.duration / 1000.0) do - buckets DURATION_BUCKET - end + def observe(histogram, event, &block) + current_transaction&.observe(histogram, event.duration / 1000.0, &block) end def current_transaction diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb index 94c5d965200..0df64f2897e 100644 --- a/lib/gitlab/metrics/subscribers/external_http.rb +++ b/lib/gitlab/metrics/subscribers/external_http.rb @@ -37,7 +37,7 @@ module Gitlab def request(event) payload = event.payload - add_to_detail_store(payload) + add_to_detail_store(event.time, payload) add_to_request_store(payload) expose_metrics(payload) end @@ -48,10 +48,11 @@ module Gitlab ::Gitlab::Metrics::Transaction.current end - def add_to_detail_store(payload) + def add_to_detail_store(start, payload) return unless Gitlab::PerformanceBar.enabled_for_request? self.class.detail_store << { + start: start, duration: payload[:duration], scheme: payload[:scheme], method: payload[:method], diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 79f1abe820f..329041e3ba2 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -31,7 +31,7 @@ module Gitlab RACK_ENV_KEY = 'HTTP_GITLAB_WORKHORSE_MULTIPART_FIELDS' JWT_PARAM_SUFFIX = '.gitlab-workhorse-upload' JWT_PARAM_FIXED_KEY = 'upload' - REWRITTEN_FIELD_NAME_MAX_LENGTH = 10000.freeze + REWRITTEN_FIELD_NAME_MAX_LENGTH = 10000 class Handler def initialize(env, message) diff --git a/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb b/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb new file mode 100644 index 00000000000..d16c068c3c0 --- /dev/null +++ b/lib/gitlab/middleware/rack_multipart_tempfile_factory.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + class RackMultipartTempfileFactory + # Immediately unlink the created temporary file so we don't have to rely + # on Rack::TempfileReaper catching this after the fact. + FACTORY = lambda do |filename, content_type| + Rack::Multipart::Parser::TEMPFILE_FACTORY.call(filename, content_type).tap(&:unlink) + end + + def initialize(app) + @app = app + end + + def call(env) + if ENV['GITLAB_TEMPFILE_IMMEDIATE_UNLINK'] == '1' + env[Rack::RACK_MULTIPART_TEMPFILE_FACTORY] = FACTORY + end + + @app.call(env) + end + end + end +end diff --git a/lib/gitlab/middleware/same_site_cookies.rb b/lib/gitlab/middleware/same_site_cookies.rb index 37ccc5abb10..405732e8015 100644 --- a/lib/gitlab/middleware/same_site_cookies.rb +++ b/lib/gitlab/middleware/same_site_cookies.rb @@ -17,7 +17,7 @@ module Gitlab module Middleware class SameSiteCookies - COOKIE_SEPARATOR = "\n".freeze + COOKIE_SEPARATOR = "\n" def initialize(app) @app = app diff --git a/lib/gitlab/object_hierarchy.rb b/lib/gitlab/object_hierarchy.rb index b1a1045a1f0..9a74266693b 100644 --- a/lib/gitlab/object_hierarchy.rb +++ b/lib/gitlab/object_hierarchy.rb @@ -68,13 +68,22 @@ module Gitlab expose_depth = hierarchy_order.present? hierarchy_order ||= :asc - recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all).distinct - # if hierarchy_order is given, the calculated `depth` should be present in SELECT if expose_depth + recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all).distinct read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: hierarchy_order)) else - read_only(remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order)) + recursive_query = base_and_ancestors_cte(upto).apply_to(model.all) + + if skip_ordering? + recursive_query = recursive_query.distinct + else + recursive_query = recursive_query.reselect(*recursive_query.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct + recursive_query = model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)) + recursive_query = remove_depth_and_maintain_order(recursive_query, hierarchy_order: hierarchy_order) + end + + read_only(recursive_query) end else recursive_query = base_and_ancestors_cte(upto, hierarchy_order).apply_to(model.all) @@ -93,12 +102,21 @@ module Gitlab def base_and_descendants(with_depth: false) if use_distinct? # Always calculate `depth`, remove it later if with_depth is false - base_cte = base_and_descendants_cte(with_depth: true).apply_to(model.all).distinct - if with_depth - read_only(model.from(Arel::Nodes::As.new(recursive_query.arel, objects_table)).order(depth: :asc)) + base_cte = base_and_descendants_cte(with_depth: true).apply_to(model.all).distinct + read_only(model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)).order(depth: :asc)) else - read_only(remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc)) + base_cte = base_and_descendants_cte.apply_to(model.all) + + if skip_ordering? + base_cte = base_cte.distinct + else + base_cte = base_cte.reselect(*base_cte.arel.projections, 'ROW_NUMBER() OVER () as depth').distinct + base_cte = model.from(Arel::Nodes::As.new(base_cte.arel, objects_table)) + base_cte = remove_depth_and_maintain_order(base_cte, hierarchy_order: :asc) + end + + read_only(base_cte) end else read_only(base_and_descendants_cte(with_depth: with_depth).apply_to(model.all)) @@ -161,7 +179,19 @@ module Gitlab # Use distinct on the Namespace queries to avoid bad planner behavior in PG11. def use_distinct? - (model <= Namespace) && options[:use_distinct] + return unless model <= Namespace + # Global use_distinct_for_all_object_hierarchy takes precedence over use_distinct_in_object_hierarchy + return true if Feature.enabled?(:use_distinct_for_all_object_hierarchy) + return options[:use_distinct] if options.key?(:use_distinct) + + false + end + + # Skips the extra ordering when using distinct on the namespace queries + def skip_ordering? + return options[:skip_ordering] if options.key?(:skip_ordering) + + false end # Remove the extra `depth` field using an INNER JOIN to avoid breaking UNION queries diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb index 33e709360ad..98e87e9e915 100644 --- a/lib/gitlab/pages.rb +++ b/lib/gitlab/pages.rb @@ -3,7 +3,7 @@ module Gitlab module Pages VERSION = File.read(Rails.root.join("GITLAB_PAGES_VERSION")).strip.freeze - INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request'.freeze + INTERNAL_API_REQUEST_HEADER = 'Gitlab-Pages-Api-Request' MAX_SIZE = 1.terabyte include JwtAuthenticatable diff --git a/lib/gitlab/pages/migration_helper.rb b/lib/gitlab/pages/migration_helper.rb new file mode 100644 index 00000000000..8f8667fafd9 --- /dev/null +++ b/lib/gitlab/pages/migration_helper.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class MigrationHelper + def initialize(logger = nil) + @logger = logger + end + + def migrate_to_remote_storage + deployments = ::PagesDeployment.with_files_stored_locally + migrate(deployments, ObjectStorage::Store::REMOTE) + end + + def migrate_to_local_storage + deployments = ::PagesDeployment.with_files_stored_remotely + migrate(deployments, ObjectStorage::Store::LOCAL) + end + + private + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i + end + + def migrate(deployments, store) + deployments.find_each(batch_size: batch_size) do |deployment| # rubocop:disable CodeReuse/ActiveRecord + deployment.file.migrate!(store) + + log_success(deployment, store) + rescue => e + log_error(e, deployment) + end + end + + def log_success(deployment, store) + logger.info("Transferred deployment ID #{deployment.id} of type #{deployment.file_type} with size #{deployment.size} to #{storage_label(store)} storage") + end + + def log_error(err, deployment) + logger.warn("Failed to transfer deployment of type #{deployment.file_type} and ID #{deployment.id} with error: #{err.message}") + end + + def storage_label(store) + if store == ObjectStorage::Store::LOCAL + 'local' + else + 'object' + end + end + end + end +end diff --git a/lib/gitlab/pages/settings.rb b/lib/gitlab/pages/settings.rb index 8650a80a85e..be71018e851 100644 --- a/lib/gitlab/pages/settings.rb +++ b/lib/gitlab/pages/settings.rb @@ -6,12 +6,28 @@ module Gitlab DiskAccessDenied = Class.new(StandardError) def path - if ::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite? - raise DiskAccessDenied - end + report_denied_disk_access super end + + def local_store + @local_store ||= ::Gitlab::Pages::Stores::LocalStore.new(super) + end + + private + + def disk_access_denied? + return true unless ::Settings.pages.local_store&.enabled + + ::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite? + end + + def report_denied_disk_access + raise DiskAccessDenied if disk_access_denied? + rescue => e + ::Gitlab::ErrorTracking.track_exception(e) + end end end end diff --git a/lib/gitlab/pages/stores/local_store.rb b/lib/gitlab/pages/stores/local_store.rb new file mode 100644 index 00000000000..68a7ebaceff --- /dev/null +++ b/lib/gitlab/pages/stores/local_store.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + module Stores + class LocalStore < ::SimpleDelegator + def enabled + return false unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) + + super + end + end + end + end +end diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb index c1ccfae3e1f..ae5539c03b1 100644 --- a/lib/gitlab/pages_transfer.rb +++ b/lib/gitlab/pages_transfer.rb @@ -12,7 +12,7 @@ module Gitlab class Async METHODS.each do |meth| define_method meth do |*args| - next unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) + next unless Settings.pages.local_store.enabled PagesTransferWorker.perform_async(meth, args) end @@ -21,7 +21,7 @@ module Gitlab METHODS.each do |meth| define_method meth do |*args| - next unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) + next unless Settings.pages.local_store.enabled super(*args) end diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index e8e68a5c4a5..e596e1bac9d 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -55,14 +55,14 @@ module Gitlab # scope :created_at_ordered, -> { # keyset_order = Gitlab::Pagination::Keyset::Order.build([ # Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - # attribute: :created_at, + # attribute_name: :created_at, # column_expression: Project.arel_table[:created_at], # order_expression: Project.arel_table[:created_at].asc, # distinct: false, # values in the column are not unique # nullable: :nulls_last # we might see NULL values (bottom) # ), # Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( - # attribute: :id, + # attribute_name: :id, # order_expression: Project.arel_table[:id].asc # ) # ]) @@ -93,7 +93,7 @@ module Gitlab end def cursor_attributes_for_node(node) - column_definitions.each_with_object({}) do |column_definition, hash| + column_definitions.each_with_object({}.with_indifferent_access) do |column_definition, hash| field_value = node[column_definition.attribute_name] hash[column_definition.attribute_name] = if field_value.is_a?(Time) field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') @@ -162,7 +162,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def apply_cursor_conditions(scope, values = {}) scope = apply_custom_projections(scope) - scope.where(build_where_values(values)) + scope.where(build_where_values(values.with_indifferent_access)) end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/pagination/offset_header_builder.rb b/lib/gitlab/pagination/offset_header_builder.rb index 32089e40932..555f0e5a607 100644 --- a/lib/gitlab/pagination/offset_header_builder.rb +++ b/lib/gitlab/pagination/offset_header_builder.rb @@ -5,9 +5,9 @@ module Gitlab class OffsetHeaderBuilder attr_reader :request_context, :per_page, :page, :next_page, :prev_page, :total, :total_pages - delegate :params, :header, :request, to: :request_context + delegate :request, to: :request_context - def initialize(request_context:, per_page:, page:, next_page:, prev_page: nil, total:, total_pages:) + def initialize(request_context:, per_page:, page:, next_page:, prev_page: nil, total: nil, total_pages: nil, params: nil) @request_context = request_context @per_page = per_page @page = page @@ -15,6 +15,7 @@ module Gitlab @prev_page = prev_page @total = total @total_pages = total_pages + @params = params end def execute(exclude_total_headers: false, data_without_counts: false) @@ -56,10 +57,24 @@ module Gitlab end def page_href(next_page_params = {}) - query_params = params.merge(**next_page_params, per_page: params[:per_page]).to_query + query_params = params.merge(**next_page_params, per_page: per_page).to_query build_page_url(query_params: query_params) end + + def params + @params || request_context.params + end + + def header(name, value) + if request_context.respond_to?(:header) + # For Grape API + request_context.header(name, value) + else + # For rails controllers + request_context.response.headers[name] = value + end + end end end end diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb index 380340b80be..c2a4602fd16 100644 --- a/lib/gitlab/performance_bar/stats.rb +++ b/lib/gitlab/performance_bar/stats.rb @@ -5,6 +5,12 @@ module Gitlab # This class fetches Peek stats stored in redis and logs them in a # structured log (so these can be then analyzed in Kibana) class Stats + IGNORED_BACKTRACE_LOCATIONS = %w[ + ee/lib/ee/peek + lib/peek + lib/gitlab/database + ].freeze + def initialize(redis) @redis = redis end @@ -53,7 +59,8 @@ module Gitlab end def parse_backtrace(backtrace) - return unless match = /(?<filename>.*):(?<filenum>\d+):in `(?<method>.*)'/.match(backtrace.first) + return unless backtrace_row = find_caller(backtrace) + return unless match = /(?<filename>.*):(?<filenum>\d+):in `(?<method>.*)'/.match(backtrace_row) { filename: match[:filename], @@ -65,6 +72,12 @@ module Gitlab } end + def find_caller(backtrace) + backtrace.find do |line| + !line.start_with?(*IGNORED_BACKTRACE_LOCATIONS) + end + end + def logger @logger ||= Gitlab::PerformanceBar::Logger.build end diff --git a/lib/gitlab/phabricator_import.rb b/lib/gitlab/phabricator_import.rb index 3885a9934d5..4c9d54a93ce 100644 --- a/lib/gitlab/phabricator_import.rb +++ b/lib/gitlab/phabricator_import.rb @@ -5,7 +5,7 @@ module Gitlab BaseError = Class.new(StandardError) def self.available? - Feature.enabled?(:phabricator_import) && + Feature.enabled?(:phabricator_import, default_enabled: :yaml) && Gitlab::CurrentSettings.import_sources.include?('phabricator') end end diff --git a/lib/gitlab/phabricator_import/issues/importer.rb b/lib/gitlab/phabricator_import/issues/importer.rb index a58438452ff..478c26af030 100644 --- a/lib/gitlab/phabricator_import/issues/importer.rb +++ b/lib/gitlab/phabricator_import/issues/importer.rb @@ -4,7 +4,8 @@ module Gitlab module Issues class Importer def initialize(project, after = nil) - @project, @after = project, after + @project = project + @after = after end def execute diff --git a/lib/gitlab/phabricator_import/issues/task_importer.rb b/lib/gitlab/phabricator_import/issues/task_importer.rb index c17f3e1729a..9c419ecb700 100644 --- a/lib/gitlab/phabricator_import/issues/task_importer.rb +++ b/lib/gitlab/phabricator_import/issues/task_importer.rb @@ -4,7 +4,8 @@ module Gitlab module Issues class TaskImporter def initialize(project, task) - @project, @task = project, task + @project = project + @task = task end def execute diff --git a/lib/gitlab/phabricator_import/project_creator.rb b/lib/gitlab/phabricator_import/project_creator.rb index b37a5b44980..c842798ca74 100644 --- a/lib/gitlab/phabricator_import/project_creator.rb +++ b/lib/gitlab/phabricator_import/project_creator.rb @@ -55,12 +55,13 @@ module Gitlab end def project_feature_attributes - @project_features_attributes ||= begin - # everything disabled except for issues - ProjectFeature::FEATURES.map do |feature| - [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED] - end.to_h.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED) - end + @project_features_attributes ||= + begin + # everything disabled except for issues + ProjectFeature::FEATURES.to_h do |feature| + [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED] + end.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED) + end end def import_data diff --git a/lib/gitlab/phabricator_import/user_finder.rb b/lib/gitlab/phabricator_import/user_finder.rb index 4b50431e0e0..c6058d12527 100644 --- a/lib/gitlab/phabricator_import/user_finder.rb +++ b/lib/gitlab/phabricator_import/user_finder.rb @@ -4,7 +4,8 @@ module Gitlab module PhabricatorImport class UserFinder def initialize(project, phids) - @project, @phids = project, phids + @project = project + @phids = phids @loaded_phids = Set.new end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 56eeea6e746..32d3eeb8cd2 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -5,7 +5,11 @@ module Gitlab attr_reader :title, :name, :description, :preview, :logo def initialize(name, title, description, preview, logo = 'illustrations/gitlab_logo.svg') - @name, @title, @description, @preview, @logo = name, title, description, preview, logo + @name = name + @title = title + @description = description + @preview = preview + @logo = logo end def file diff --git a/lib/gitlab/prometheus/adapter.rb b/lib/gitlab/prometheus/adapter.rb index ed10ef2917f..76e65d29c7a 100644 --- a/lib/gitlab/prometheus/adapter.rb +++ b/lib/gitlab/prometheus/adapter.rb @@ -19,6 +19,10 @@ module Gitlab end def cluster_prometheus_adapter + if cluster&.integration_prometheus + return cluster.integration_prometheus + end + application = cluster&.application_prometheus application if application&.available? diff --git a/lib/gitlab/prometheus/queries/matched_metric_query.rb b/lib/gitlab/prometheus/queries/matched_metric_query.rb index e4d44df3baf..73de5a11998 100644 --- a/lib/gitlab/prometheus/queries/matched_metric_query.rb +++ b/lib/gitlab/prometheus/queries/matched_metric_query.rb @@ -4,7 +4,7 @@ module Gitlab module Prometheus module Queries class MatchedMetricQuery < BaseQuery - MAX_QUERY_ITEMS = 40.freeze + MAX_QUERY_ITEMS = 40 def query groups_data.map do |group, data| diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index 965349ad711..0fcf63d03fc 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -140,7 +140,7 @@ module Gitlab end def mapped_options - options.keys.map { |k| [gitlab_http_key(k), options[k]] }.to_h + options.keys.to_h { |k| [gitlab_http_key(k), options[k]] } end def http_options diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 02446a7953b..ce9fced9465 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -5,6 +5,7 @@ module Gitlab VALID_OPTIONS = HashWithIndifferentAccess.new({ merge_request: { keys: [ + :assign, :create, :description, :label, @@ -12,6 +13,7 @@ module Gitlab :remove_source_branch, :target, :title, + :unassign, :unlabel ] }, @@ -23,7 +25,9 @@ module Gitlab MULTI_VALUE_OPTIONS = [ %w[ci variable], %w[merge_request label], - %w[merge_request unlabel] + %w[merge_request unlabel], + %w[merge_request assign], + %w[merge_request unassign] ].freeze NAMESPACE_ALIASES = HashWithIndifferentAccess.new({ diff --git a/lib/gitlab/query_limiting.rb b/lib/gitlab/query_limiting.rb index 5e46e26e14e..03386dca141 100644 --- a/lib/gitlab/query_limiting.rb +++ b/lib/gitlab/query_limiting.rb @@ -6,28 +6,36 @@ module Gitlab # # This is only enabled in development and test to ensure we don't produce # any errors that users of other environments can't do anything about themselves. - def self.enable? + def self.enabled_for_env? Rails.env.development? || Rails.env.test? end + def self.enabled? + enabled_for_env? && + !Gitlab::SafeRequestStore[:query_limiting_disabled] + end + # Allows the current request to execute any number of SQL queries. # # This method should _only_ be used when there's a corresponding issue to # reduce the number of queries. # # The issue URL is only meant to push developers into creating an issue - # instead of blindly whitelisting offending blocks of code. - def self.whitelist(issue_url) - return unless enable? - + # instead of blindly disabling for offending blocks of code. + def self.disable!(issue_url) unless issue_url.start_with?('https://') raise( ArgumentError, - 'You must provide a valid issue URL in order to whitelist a block of code' + 'You must provide a valid issue URL in order to allow a block of code' ) end - Transaction&.current&.whitelisted = true + Gitlab::SafeRequestStore[:query_limiting_disabled] = true + end + + # Enables query limiting for the request. + def self.enable! + Gitlab::SafeRequestStore[:query_limiting_disabled] = nil end end end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index 196072dddda..643b2540c37 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -5,7 +5,7 @@ module Gitlab class Transaction THREAD_KEY = :__gitlab_query_counts_transaction - attr_accessor :count, :whitelisted + attr_accessor :count # The name of the action (e.g. `UsersController#show`) that is being # executed. @@ -45,7 +45,6 @@ module Gitlab def initialize @action = nil @count = 0 - @whitelisted = false @sql_executed = [] end @@ -59,7 +58,7 @@ module Gitlab end def increment - @count += 1 unless whitelisted + @count += 1 if enabled? end def executed_sql(sql) @@ -83,6 +82,10 @@ module Gitlab ["#{header}: #{msg}", log, ellipsis].compact.join("\n") end + + def enabled? + ::Gitlab::QueryLimiting.enabled? + end end end end diff --git a/lib/gitlab/quick_actions/command_definition.rb b/lib/gitlab/quick_actions/command_definition.rb index b17a0208f95..8ce13db4c03 100644 --- a/lib/gitlab/quick_actions/command_definition.rb +++ b/lib/gitlab/quick_actions/command_definition.rb @@ -56,15 +56,18 @@ module Gitlab end def execute(context, arg) - return unless executable?(context) + return if noop? count_commands_executed_in(context) + return unless available?(context) + execute_block(action_block, context, arg) end def execute_message(context, arg) - return unless executable?(context) + return if noop? + return _('Could not apply %{name} command.') % { name: name } unless available?(context) if execution_message.respond_to?(:call) execute_block(execution_message, context, arg) @@ -101,10 +104,6 @@ module Gitlab private - def executable?(context) - !noop? && available?(context) - end - def count_commands_executed_in(context) return unless context.respond_to?(:commands_executed_count=) diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index 4934c12a339..b7d58e05651 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -182,7 +182,7 @@ module Gitlab parse_params do |raw_time_date| Gitlab::QuickActions::SpendTimeAndDateSeparator.new(raw_time_date).execute end - command :spend do |time_spent, time_spent_date| + command :spend, :spent do |time_spent, time_spent_date| if time_spent @updates[:spend_time] = { duration: time_spent, diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index 67e3a5de223..bd6d2e016b4 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -34,12 +34,16 @@ module Gitlab path =~ %r{^/-/(health|liveness|readiness|metrics)} end + def container_registry_event? + path =~ %r{^/api/v\d+/container_registry_event/} + end + def product_analytics_collector_request? path.start_with?('/-/collector/i') end def should_be_skipped? - api_internal_request? || health_check_request? + api_internal_request? || health_check_request? || container_registry_event? end def web_request? diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 00739c05386..488ba04f87c 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -181,7 +181,7 @@ module Gitlab end def generic_package_version_regex - /\A\d+\.\d+\.\d+\z/ + maven_version_regex end def generic_package_name_regex @@ -385,11 +385,11 @@ module Gitlab end def merge_request_wip - /(?i)(\[WIP\]\s*|WIP:\s*|WIP$)/ + /(?i)(\[WIP\]\s*|WIP:\s*|\AWIP\z)/ end def merge_request_draft - /(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft$)/ + /\A(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft\z)/ end def issue diff --git a/lib/gitlab/relative_positioning/closed_range.rb b/lib/gitlab/relative_positioning/closed_range.rb index 8916d1face5..11fba05edee 100644 --- a/lib/gitlab/relative_positioning/closed_range.rb +++ b/lib/gitlab/relative_positioning/closed_range.rb @@ -4,7 +4,8 @@ module Gitlab module RelativePositioning class ClosedRange < RelativePositioning::Range def initialize(lhs, rhs) - @lhs, @rhs = lhs, rhs + @lhs = lhs + @rhs = rhs raise IllegalRange, 'Either lhs or rhs is missing' unless lhs && rhs raise IllegalRange, 'lhs and rhs cannot be the same object' if lhs == rhs end diff --git a/lib/gitlab/relative_positioning/gap.rb b/lib/gitlab/relative_positioning/gap.rb index ab894141a60..2e30e598eb0 100644 --- a/lib/gitlab/relative_positioning/gap.rb +++ b/lib/gitlab/relative_positioning/gap.rb @@ -6,7 +6,8 @@ module Gitlab attr_reader :start_pos, :end_pos def initialize(start_pos, end_pos) - @start_pos, @end_pos = start_pos, end_pos + @start_pos = start_pos + @end_pos = end_pos end def ==(other) diff --git a/lib/gitlab/repository_cache_adapter.rb b/lib/gitlab/repository_cache_adapter.rb index eb7c9bccf96..d0230c035cc 100644 --- a/lib/gitlab/repository_cache_adapter.rb +++ b/lib/gitlab/repository_cache_adapter.rb @@ -60,14 +60,17 @@ module Gitlab define_method("#{name}_include?") do |value| ivar = "@#{name}_include" memoized = instance_variable_get(ivar) || {} + lookup = proc { __send__(name).include?(value) } # rubocop:disable GitlabSecurity/PublicSend next memoized[value] if memoized.key?(value) memoized[value] = - if strong_memoized?(name) || !redis_set_cache.exist?(name) - __send__(name).include?(value) # rubocop:disable GitlabSecurity/PublicSend + if strong_memoized?(name) + lookup.call else - redis_set_cache.include?(name, value) + result, exists = redis_set_cache.try_include?(name, value) + + exists ? result : lookup.call end instance_variable_set(ivar, memoized)[value] diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb index d479d3115a6..430f3e8d162 100644 --- a/lib/gitlab/repository_hash_cache.rb +++ b/lib/gitlab/repository_hash_cache.rb @@ -148,7 +148,7 @@ module Gitlab # @param hash [Hash] # @return [Hash] the stringified hash def standardize_hash(hash) - hash.map { |k, v| [k.to_s, v.to_s] }.to_h + hash.to_h { |k, v| [k.to_s, v.to_s] } end # Record metrics in Prometheus. diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index 69c1688767c..f73ac628bce 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -36,10 +36,32 @@ module Gitlab end def fetch(key, &block) - if exist?(key) - read(key) - else - write(key, yield) + full_key = cache_key(key) + + smembers, exists = with do |redis| + redis.multi do + redis.smembers(full_key) + redis.exists(full_key) + end + end + + return smembers if exists + + write(key, yield) + end + + # Searches the cache set using SSCAN with the MATCH option. The MATCH + # parameter is the pattern argument. + # See https://redis.io/commands/scan#the-match-option for more information. + # Returns an Enumerator that enumerates all SSCAN hits. + def search(key, pattern, &block) + full_key = cache_key(key) + + with do |redis| + exists = redis.exists(full_key) + write(key, yield) unless exists + + redis.sscan_each(full_key, match: pattern) end end end diff --git a/lib/gitlab/search_context.rb b/lib/gitlab/search_context.rb index c3bb0ff26f2..0323220690a 100644 --- a/lib/gitlab/search_context.rb +++ b/lib/gitlab/search_context.rb @@ -129,7 +129,10 @@ module Gitlab 'wiki_blobs' elsif view_context.current_controller?(:commits) 'commits' - else nil + elsif view_context.current_controller?(:groups) + if %w(issues merge_requests).include?(view_context.controller.action_name) + view_context.controller.action_name + end end end end @@ -160,3 +163,5 @@ module Gitlab end end end + +Gitlab::SearchContext::Builder.prepend_ee_mod diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 591265d014e..0f2b7b194c9 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -51,6 +51,19 @@ module Gitlab with { |redis| redis.sismember(cache_key(key), value) } end + # Like include?, but also tells us if the cache was populated when it ran + # by returning two booleans: [member_exists, set_exists] + def try_include?(key, value) + full_key = cache_key(key) + + with do |redis| + redis.multi do + redis.sismember(full_key, value) + redis.exists(full_key) + end + end + end + def ttl(key) with { |redis| redis.ttl(cache_key(key)) } end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 7561e36cc33..3ac20724403 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -98,6 +98,10 @@ module Gitlab if Rails.env.test? socket_filename = options[:gitaly_socket] || "gitaly.socket" + prometheus_listen_addr = options[:prometheus_listen_addr] + + git_bin_path = File.expand_path('../gitaly/_build/deps/git/install/bin/git') + git_bin_path = nil unless File.exist?(git_bin_path) config = { # Override the set gitaly_address since Praefect is in the loop @@ -106,8 +110,12 @@ module Gitlab # Compared to production, tests run in constrained environments. This # number is meant to grow with the number of concurrent rails requests / # sidekiq jobs, and concurrency will be low anyway in test. - git: { catfile_cache_size: 5 } - } + git: { + catfile_cache_size: 5, + bin_path: git_bin_path + }.compact, + prometheus_listen_addr: prometheus_listen_addr + }.compact storage_path = Rails.root.join('tmp', 'tests', 'second_storage').to_s storages << { name: 'test_second_storage', path: storage_path } diff --git a/lib/gitlab/sidekiq_cluster/cli.rb b/lib/gitlab/sidekiq_cluster/cli.rb index e471517c50a..9490d543dd1 100644 --- a/lib/gitlab/sidekiq_cluster/cli.rb +++ b/lib/gitlab/sidekiq_cluster/cli.rb @@ -53,11 +53,11 @@ module Gitlab 'You cannot specify --queue-selector and --experimental-queue-selector together' end - all_queues = SidekiqConfig::CliMethods.all_queues(@rails_path) - queue_names = SidekiqConfig::CliMethods.worker_queues(@rails_path) + worker_metadatas = SidekiqConfig::CliMethods.worker_metadatas(@rails_path) + worker_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path) - queue_groups = argv.map do |queues| - next queue_names if queues == '*' + queue_groups = argv.map do |queues_or_query_string| + next worker_queues if queues_or_query_string == SidekiqConfig::WorkerMatcher::WILDCARD_MATCH # When using the queue query syntax, we treat each queue group # as a worker attribute query, and resolve the queues for the @@ -65,14 +65,14 @@ module Gitlab # Simplify with https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/646 if @queue_selector || @experimental_queue_selector - SidekiqConfig::CliMethods.query_workers(queues, all_queues) + SidekiqConfig::CliMethods.query_queues(queues_or_query_string, worker_metadatas) else - SidekiqConfig::CliMethods.expand_queues(queues.split(','), queue_names) + SidekiqConfig::CliMethods.expand_queues(queues_or_query_string.split(','), worker_queues) end end if @negate_queues - queue_groups.map! { |queues| queue_names - queues } + queue_groups.map! { |queues| worker_queues - queues } end if queue_groups.all?(&:empty?) diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 633291dcdf3..78d45b5f3f0 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -13,10 +13,17 @@ module Gitlab (EE_QUEUE_CONFIG_PATH if Gitlab.ee?) ].compact.freeze - DEFAULT_WORKERS = [ - DummyWorker.new('default', weight: 1, tags: []), - DummyWorker.new('mailers', weight: 2, tags: []) - ].map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze + # This maps workers not in our application code to queues. We need + # these queues in our YAML files to ensure we don't accidentally + # miss jobs from these queues. + # + # The default queue should be unused, which is why it maps to an + # invalid class name. We keep it in the YAML file for safety, just + # in case anything does get scheduled to run there. + DEFAULT_WORKERS = { + '_' => DummyWorker.new('default', weight: 1, tags: []), + 'ActionMailer::MailDeliveryJob' => DummyWorker.new('mailers', feature_category: :issue_tracking, urgency: 'low', weight: 2, tags: []) + }.transform_values { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze class << self include Gitlab::SidekiqConfig::CliMethods @@ -40,7 +47,7 @@ module Gitlab def workers @workers ||= begin result = [] - result.concat(DEFAULT_WORKERS) + result.concat(DEFAULT_WORKERS.values) result.concat(find_workers(Rails.root.join('app', 'workers'), ee: false)) if Gitlab.ee? diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb index a256632bc12..8eef15f9ccb 100644 --- a/lib/gitlab/sidekiq_config/cli_methods.rb +++ b/lib/gitlab/sidekiq_config/cli_methods.rb @@ -12,35 +12,19 @@ module Gitlab # rubocop:disable Gitlab/ModuleWithInstanceVariables extend self + # The file names are misleading. Those files contain the metadata of the + # workers. They should be renamed to all_workers instead. + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1018 QUEUE_CONFIG_PATHS = begin result = %w[app/workers/all_queues.yml] result << 'ee/app/workers/all_queues.yml' if Gitlab.ee? result end.freeze - QUERY_OR_OPERATOR = '|' - QUERY_AND_OPERATOR = '&' - QUERY_CONCATENATE_OPERATOR = ',' - QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze + def worker_metadatas(rails_path = Rails.root.to_s) + @worker_metadatas ||= {} - QUERY_PREDICATES = { - feature_category: :to_sym, - has_external_dependencies: lambda { |value| value == 'true' }, - name: :to_s, - resource_boundary: :to_sym, - tags: :to_sym, - urgency: :to_sym - }.freeze - - QueryError = Class.new(StandardError) - InvalidTerm = Class.new(QueryError) - UnknownOperator = Class.new(QueryError) - UnknownPredicate = Class.new(QueryError) - - def all_queues(rails_path = Rails.root.to_s) - @worker_queues ||= {} - - @worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path| + @worker_metadatas[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path| full_path = File.join(rails_path, path) File.exist?(full_path) ? YAML.load_file(full_path) : [] @@ -49,7 +33,7 @@ module Gitlab # rubocop:enable Gitlab/ModuleWithInstanceVariables def worker_queues(rails_path = Rails.root.to_s) - worker_names(all_queues(rails_path)) + worker_names(worker_metadatas(rails_path)) end def expand_queues(queues, all_queues = self.worker_queues) @@ -62,13 +46,18 @@ module Gitlab end end - def query_workers(query_string, queues) - worker_names(queues.select(&query_string_to_lambda(query_string))) + def query_queues(query_string, worker_metadatas) + matcher = SidekiqConfig::WorkerMatcher.new(query_string) + selected_metadatas = worker_metadatas.select do |worker_metadata| + matcher.match?(worker_metadata) + end + + worker_names(selected_metadatas) end def clear_memoization! - if instance_variable_defined?('@worker_queues') - remove_instance_variable('@worker_queues') + if instance_variable_defined?('@worker_metadatas') + remove_instance_variable('@worker_metadatas') end end @@ -77,53 +66,6 @@ module Gitlab def worker_names(workers) workers.map { |queue| queue[:name] } end - - def query_string_to_lambda(query_string) - or_clauses = query_string.split(QUERY_OR_OPERATOR).map do |and_clauses_string| - and_clauses_predicates = and_clauses_string.split(QUERY_AND_OPERATOR).map do |term| - predicate_for_term(term) - end - - lambda { |worker| and_clauses_predicates.all? { |predicate| predicate.call(worker) } } - end - - lambda { |worker| or_clauses.any? { |predicate| predicate.call(worker) } } - end - - def predicate_for_term(term) - match = term.match(QUERY_TERM_REGEX) - - raise InvalidTerm.new("Invalid term: #{term}") unless match - - _, lhs, op, rhs = *match - - predicate_for_op(op, predicate_factory(lhs, rhs.split(QUERY_CONCATENATE_OPERATOR))) - end - - def predicate_for_op(op, predicate) - case op - when '=' - predicate - when '!=' - lambda { |worker| !predicate.call(worker) } - else - # This is unreachable because InvalidTerm will be raised instead, but - # keeping it allows to guard against that changing in future. - raise UnknownOperator.new("Unknown operator: #{op}") - end - end - - def predicate_factory(lhs, values) - values_block = QUERY_PREDICATES[lhs.to_sym] - - raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block - - lambda do |queue| - comparator = Array(queue[lhs.to_sym]).to_set - - values.map(&values_block).to_set.intersect?(comparator) - end - end end end end diff --git a/lib/gitlab/sidekiq_config/worker_matcher.rb b/lib/gitlab/sidekiq_config/worker_matcher.rb new file mode 100644 index 00000000000..fe5ac10c65a --- /dev/null +++ b/lib/gitlab/sidekiq_config/worker_matcher.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqConfig + class WorkerMatcher + WILDCARD_MATCH = '*' + QUERY_OR_OPERATOR = '|' + QUERY_AND_OPERATOR = '&' + QUERY_CONCATENATE_OPERATOR = ',' + QUERY_TERM_REGEX = %r{^(\w+)(!?=)([\w:#{QUERY_CONCATENATE_OPERATOR}]+)}.freeze + + QUERY_PREDICATES = { + feature_category: :to_sym, + has_external_dependencies: lambda { |value| value == 'true' }, + name: :to_s, + resource_boundary: :to_sym, + tags: :to_sym, + urgency: :to_sym + }.freeze + + QueryError = Class.new(StandardError) + InvalidTerm = Class.new(QueryError) + UnknownOperator = Class.new(QueryError) + UnknownPredicate = Class.new(QueryError) + + def initialize(query_string) + @match_lambda = query_string_to_lambda(query_string) + end + + def match?(worker_metadata) + @match_lambda.call(worker_metadata) + end + + private + + def query_string_to_lambda(query_string) + return lambda { |_worker| true } if query_string.strip == WILDCARD_MATCH + + or_clauses = query_string.split(QUERY_OR_OPERATOR).map do |and_clauses_string| + and_clauses_predicates = and_clauses_string.split(QUERY_AND_OPERATOR).map do |term| + predicate_for_term(term) + end + + lambda { |worker| and_clauses_predicates.all? { |predicate| predicate.call(worker) } } + end + + lambda { |worker| or_clauses.any? { |predicate| predicate.call(worker) } } + end + + def predicate_for_term(term) + match = term.match(QUERY_TERM_REGEX) + + raise InvalidTerm.new("Invalid term: #{term}") unless match + + _, lhs, op, rhs = *match + + predicate_for_op(op, predicate_factory(lhs, rhs.split(QUERY_CONCATENATE_OPERATOR))) + end + + def predicate_for_op(op, predicate) + case op + when '=' + predicate + when '!=' + lambda { |worker| !predicate.call(worker) } + else + # This is unreachable because InvalidTerm will be raised instead, but + # keeping it allows to guard against that changing in future. + raise UnknownOperator.new("Unknown operator: #{op}") + end + end + + def predicate_factory(lhs, values) + values_block = QUERY_PREDICATES[lhs.to_sym] + + raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block + + lambda do |queue| + comparator = Array(queue[lhs.to_sym]).to_set + + values.map(&values_block).to_set.intersect?(comparator) + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 654b17c5740..b1fb3771c78 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -39,9 +39,7 @@ module Gitlab private def add_instrumentation_keys!(job, output_payload) - instrumentation_values = job.slice(*::Gitlab::InstrumentationHelper.keys).stringify_keys - - output_payload.merge!(instrumentation_values) + output_payload.merge!(job[:instrumentation].stringify_keys) end def add_logging_extras!(job, output_payload) diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index a2696e17078..563a105484d 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -43,3 +43,5 @@ module Gitlab end end end + +Gitlab::SidekiqMiddleware.singleton_class.prepend_if_ee('EE::Gitlab::SidekiqMiddleware') diff --git a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb index 36204e1bee0..1b33743a0e9 100644 --- a/lib/gitlab/sidekiq_middleware/admin_mode/client.rb +++ b/lib/gitlab/sidekiq_middleware/admin_mode/client.rb @@ -8,7 +8,8 @@ module Gitlab # If enabled then it injects a job field that persists through the job execution class Client def call(_worker_class, job, _queue, _redis_pool) - return yield unless ::Feature.enabled?(:user_mode_in_session) + # Not calling Gitlab::CurrentSettings.admin_mode on purpose on sidekiq middleware + # Only when admin mode application setting is enabled might the admin_mode_user_id be non-nil here # Admin mode enabled in the original request or in a nested sidekiq job admin_mode_user_id = find_admin_user_id diff --git a/lib/gitlab/sidekiq_middleware/admin_mode/server.rb b/lib/gitlab/sidekiq_middleware/admin_mode/server.rb index 6366867a0fa..c4e64705d6e 100644 --- a/lib/gitlab/sidekiq_middleware/admin_mode/server.rb +++ b/lib/gitlab/sidekiq_middleware/admin_mode/server.rb @@ -5,7 +5,8 @@ module Gitlab module AdminMode class Server def call(_worker, job, _queue) - return yield unless Feature.enabled?(:user_mode_in_session) + # Not calling Gitlab::CurrentSettings.admin_mode on purpose on sidekiq middleware + # Only when admin_mode setting is enabled can it be true here admin_mode_user_id = job['admin_mode_user_id'] diff --git a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb index a66a4de4655..b542aa4fe4c 100644 --- a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb +++ b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb @@ -3,6 +3,24 @@ module Gitlab module SidekiqMiddleware class InstrumentationLogger + def self.keys + @keys ||= [ + :cpu_s, + :gitaly_calls, + :gitaly_duration_s, + :rugged_calls, + :rugged_duration_s, + :elasticsearch_calls, + :elasticsearch_duration_s, + :elasticsearch_timed_out_count, + *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values, + *::Gitlab::Instrumentation::Redis.known_payload_keys, + *::Gitlab::Metrics::Subscribers::ActiveRecord.known_payload_keys, + *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS, + *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS + ] + end + def call(worker, job, queue) ::Gitlab::InstrumentationHelper.init_instrumentation_data @@ -17,7 +35,10 @@ module Gitlab # because Sidekiq keeps a pristine copy of the original hash # before sending it to the middleware: # https://github.com/mperham/sidekiq/blob/53bd529a0c3f901879925b8390353129c465b1f2/lib/sidekiq/processor.rb#L115-L118 - ::Gitlab::InstrumentationHelper.add_instrumentation_data(job) + job[:instrumentation] = {}.tap do |instrumentation_values| + ::Gitlab::InstrumentationHelper.add_instrumentation_data(instrumentation_values) + instrumentation_values.slice!(*self.class.keys) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/metrics_helper.rb b/lib/gitlab/sidekiq_middleware/metrics_helper.rb index 60e79ee1188..66930a34319 100644 --- a/lib/gitlab/sidekiq_middleware/metrics_helper.rb +++ b/lib/gitlab/sidekiq_middleware/metrics_helper.rb @@ -10,6 +10,7 @@ module Gitlab def create_labels(worker_class, queue, job) worker_name = (job['wrapped'].presence || worker_class).to_s + worker = find_worker(worker_name, worker_class) labels = { queue: queue.to_s, worker: worker_name, @@ -18,15 +19,15 @@ module Gitlab feature_category: "", boundary: "" } - return labels unless worker_class && worker_class.include?(WorkerAttributes) + return labels unless worker.respond_to?(:get_urgency) - labels[:urgency] = worker_class.get_urgency.to_s - labels[:external_dependencies] = bool_as_label(worker_class.worker_has_external_dependencies?) + labels[:urgency] = worker.get_urgency.to_s + labels[:external_dependencies] = bool_as_label(worker.worker_has_external_dependencies?) - feature_category = worker_class.get_feature_category + feature_category = worker.get_feature_category labels[:feature_category] = feature_category.to_s - resource_boundary = worker_class.get_worker_resource_boundary + resource_boundary = worker.get_worker_resource_boundary labels[:boundary] = resource_boundary == :unknown ? "" : resource_boundary.to_s labels @@ -35,6 +36,10 @@ module Gitlab def bool_as_label(value) value ? TRUE_LABEL : FALSE_LABEL end + + def find_worker(worker_name, worker_class) + Gitlab::SidekiqConfig::DEFAULT_WORKERS.fetch(worker_name, worker_class) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index cf768811ffd..f5fee8050ac 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -21,6 +21,16 @@ module Gitlab Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME labels = create_labels(worker.class, queue, job) + instrument(job, labels) do + yield + end + end + + protected + + attr_reader :metrics + + def instrument(job, labels) queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration @@ -50,19 +60,18 @@ module Gitlab # job_status: done, fail match the job_status attribute in structured logging labels[:job_status] = job_succeeded ? "done" : "fail" + instrumentation = job[:instrumentation] || {} @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time) @metrics[:sidekiq_jobs_db_seconds].observe(labels, ActiveRecord::LogSubscriber.runtime / 1000) - @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(job)) - @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(job)) - @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(job)) - @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(job)) - @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(job)) + @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(instrumentation)) + @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(instrumentation)) + @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(instrumentation)) + @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(instrumentation)) + @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(instrumentation)) end end - private - def init_metrics { sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), @@ -81,29 +90,33 @@ module Gitlab } end + private + def get_thread_cputime defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 end - def get_redis_time(job) - job.fetch(:redis_duration_s, 0) + def get_redis_time(payload) + payload.fetch(:redis_duration_s, 0) end - def get_redis_calls(job) - job.fetch(:redis_calls, 0) + def get_redis_calls(payload) + payload.fetch(:redis_calls, 0) end - def get_elasticsearch_time(job) - job.fetch(:elasticsearch_duration_s, 0) + def get_elasticsearch_time(payload) + payload.fetch(:elasticsearch_duration_s, 0) end - def get_elasticsearch_calls(job) - job.fetch(:elasticsearch_calls, 0) + def get_elasticsearch_calls(payload) + payload.fetch(:elasticsearch_calls, 0) end - def get_gitaly_time(job) - job.fetch(:gitaly_duration_s, 0) + def get_gitaly_time(payload) + payload.fetch(:gitaly_duration_s, 0) end end end end + +Gitlab::SidekiqMiddleware::ServerMetrics.prepend_if_ee('EE::Gitlab::SidekiqMiddleware::ServerMetrics') diff --git a/lib/gitlab/sidekiq_queue.rb b/lib/gitlab/sidekiq_queue.rb index 807c27a71ff..4b71dfc0c1b 100644 --- a/lib/gitlab/sidekiq_queue.rb +++ b/lib/gitlab/sidekiq_queue.rb @@ -21,7 +21,7 @@ module Gitlab job_search_metadata = search_metadata .stringify_keys - .slice(*Labkit::Context::KNOWN_KEYS) + .slice(*Gitlab::ApplicationContext::KNOWN_KEYS) .transform_keys { |key| "meta.#{key}" } .compact diff --git a/lib/gitlab/slash_commands/base_command.rb b/lib/gitlab/slash_commands/base_command.rb index fcc120112f2..e184afa0032 100644 --- a/lib/gitlab/slash_commands/base_command.rb +++ b/lib/gitlab/slash_commands/base_command.rb @@ -36,7 +36,9 @@ module Gitlab attr_accessor :project, :current_user, :params, :chat_name def initialize(project, chat_name, params = {}) - @project, @current_user, @params = project, chat_name.user, params.dup + @project = project + @current_user = chat_name.user + @params = params.dup @chat_name = chat_name end diff --git a/lib/gitlab/slash_commands/presenters/issue_new.rb b/lib/gitlab/slash_commands/presenters/issue_new.rb index 552456f5836..8841fef702e 100644 --- a/lib/gitlab/slash_commands/presenters/issue_new.rb +++ b/lib/gitlab/slash_commands/presenters/issue_new.rb @@ -12,16 +12,18 @@ module Gitlab private - def fallback_message - "New issue #{issue.to_reference}: #{issue.title}" + def pretext + "I created an issue on #{author_profile_link}'s behalf: *#{issue_link}* in #{project_link}" end - def fields_with_markdown - %i(title pretext text fields) + def issue_link + "[#{issue.to_reference}](#{project_issue_url(issue.project, issue)})" end - def pretext - "I created an issue on #{author_profile_link}'s behalf: *#{issue.to_reference}* in #{project_link}" + def response_message(custom_pretext: pretext) + { + text: pretext + } end end end diff --git a/lib/gitlab/slash_commands/run.rb b/lib/gitlab/slash_commands/run.rb index 10a545e28ac..40fd7ee4f20 100644 --- a/lib/gitlab/slash_commands/run.rb +++ b/lib/gitlab/slash_commands/run.rb @@ -5,7 +5,7 @@ module Gitlab # Slash command for triggering chatops jobs. class Run < BaseCommand def self.match(text) - /\Arun\s+(?<command>\S+)(\s+(?<arguments>.+))?\z/.match(text) + /\Arun\s+(?<command>\S+)(\s+(?<arguments>.+))?\z/m.match(text) end def self.help_message diff --git a/lib/gitlab/slug/environment.rb b/lib/gitlab/slug/environment.rb index 1b87d3bb626..fd70def8e7c 100644 --- a/lib/gitlab/slug/environment.rb +++ b/lib/gitlab/slug/environment.rb @@ -26,16 +26,13 @@ module Gitlab # Repeated dashes are invalid (OpenShift limitation) slugified.squeeze!('-') - slugified = - if slugified.size > 24 || slugified != name - # Maximum length: 24 characters (OpenShift limitation) - shorten_and_add_suffix(slugified) - else - # Cannot end with a dash (Kubernetes label limitation) - slugified.chomp('-') - end - - slugified + if slugified.size > 24 || slugified != name + # Maximum length: 24 characters (OpenShift limitation) + shorten_and_add_suffix(slugified) + else + # Cannot end with a dash (Kubernetes label limitation) + slugified.chomp('-') + end end private diff --git a/lib/gitlab/sql/cte.rb b/lib/gitlab/sql/cte.rb index 7817a2a1ce2..8f37602aeaa 100644 --- a/lib/gitlab/sql/cte.rb +++ b/lib/gitlab/sql/cte.rb @@ -15,20 +15,27 @@ module Gitlab # Namespace # with(cte.to_arel). # from(cte.alias_to(ns)) + # + # To skip materialization of the CTE query by passing materialized: false + # More context: https://www.postgresql.org/docs/12/queries-with.html + # + # cte = CTE.new(:my_cte_name, materialized: false) + # class CTE attr_reader :table, :query # name - The name of the CTE as a String or Symbol. - def initialize(name, query) + def initialize(name, query, materialized: true) @table = Arel::Table.new(name) @query = query + @materialized = materialized end # Returns the Arel relation for this CTE. def to_arel sql = Arel::Nodes::SqlLiteral.new("(#{query.to_sql})") - Arel::Nodes::As.new(table, sql) + Gitlab::Database::AsWithMaterialized.new(table, sql, materialized: @materialized) end # Returns an "AS" statement that aliases the CTE name as the given table diff --git a/lib/gitlab/sql/recursive_cte.rb b/lib/gitlab/sql/recursive_cte.rb index e45ac5d4765..607ce10d778 100644 --- a/lib/gitlab/sql/recursive_cte.rb +++ b/lib/gitlab/sql/recursive_cte.rb @@ -23,9 +23,11 @@ module Gitlab attr_reader :table # name - The name of the CTE as a String or Symbol. - def initialize(name) + # union_args - The arguments supplied to Gitlab::SQL::Union class when building inner recursive query + def initialize(name, union_args: {}) @table = Arel::Table.new(name) @queries = [] + @union_args = union_args end # Adds a query to the body of the CTE. @@ -37,7 +39,7 @@ module Gitlab # Returns the Arel relation for this CTE. def to_arel - sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries).to_sql) + sql = Arel::Nodes::SqlLiteral.new(Union.new(@queries, **@union_args).to_sql) Arel::Nodes::As.new(table, Arel::Nodes::Grouping.new(sql)) end diff --git a/lib/gitlab/sql/set_operator.rb b/lib/gitlab/sql/set_operator.rb index d58a1415493..59a808eafa9 100644 --- a/lib/gitlab/sql/set_operator.rb +++ b/lib/gitlab/sql/set_operator.rb @@ -8,6 +8,9 @@ module Gitlab # ORDER BYs are dropped from the relations as the final sort order is not # guaranteed any way. # + # remove_order: false option can be used in special cases where the + # ORDER BY is necessary for the query. + # # Example usage: # # union = Gitlab::SQL::Union.new([user.personal_projects, user.projects]) @@ -15,9 +18,10 @@ module Gitlab # # Project.where("id IN (#{sql})") class SetOperator - def initialize(relations, remove_duplicates: true) + def initialize(relations, remove_duplicates: true, remove_order: true) @relations = relations @remove_duplicates = remove_duplicates + @remove_order = remove_order end def self.operator_keyword @@ -30,7 +34,9 @@ module Gitlab # By using "unprepared_statements" we remove the usage of placeholders # (thus fixing this problem), at a slight performance cost. fragments = ActiveRecord::Base.connection.unprepared_statement do - relations.map { |rel| rel.reorder(nil).to_sql }.reject(&:blank?) + relations.map do |rel| + remove_order ? rel.reorder(nil).to_sql : rel.to_sql + end.reject(&:blank?) end if fragments.any? @@ -47,7 +53,7 @@ module Gitlab private - attr_reader :relations, :remove_duplicates + attr_reader :relations, :remove_duplicates, :remove_order end end end diff --git a/lib/gitlab/sql/union.rb b/lib/gitlab/sql/union.rb index 7fb3487a5e5..c4e95284c50 100644 --- a/lib/gitlab/sql/union.rb +++ b/lib/gitlab/sql/union.rb @@ -4,8 +4,8 @@ module Gitlab module SQL # Class for building SQL UNION statements. # - # ORDER BYs are dropped from the relations as the final sort order is not - # guaranteed any way. + # By default ORDER BYs are dropped from the relations as the final sort + # order is not guaranteed any way. # # Example usage: # diff --git a/lib/gitlab/static_site_editor/config/file_config.rb b/lib/gitlab/static_site_editor/config/file_config.rb index 315c603c1dd..4180f6ccf00 100644 --- a/lib/gitlab/static_site_editor/config/file_config.rb +++ b/lib/gitlab/static_site_editor/config/file_config.rb @@ -28,7 +28,7 @@ module Gitlab def to_hash_with_defaults # NOTE: The current approach of simply mapping all the descendents' keys and values ('config') # into a flat hash may need to be enhanced as we add more complex, non-scalar entries. - @global.descendants.map { |descendant| [descendant.key, descendant.config] }.to_h + @global.descendants.to_h { |descendant| [descendant.key, descendant.config] } end private diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index a918e7bec80..3072210d7c8 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -6,6 +6,11 @@ module Gitlab ::Gitlab.dev_or_test_env? ? 'https://customers.stg.gitlab.com' : 'https://customers.gitlab.com' end - SUBSCRIPTIONS_URL = ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url).freeze + def self.subscriptions_url + ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url) + end end end + +Gitlab::SubscriptionPortal.prepend_if_jh('JH::Gitlab::SubscriptionPortal') +Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL = Gitlab::SubscriptionPortal.subscriptions_url.freeze diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index dc006877129..31e11f73fe7 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -130,10 +130,10 @@ module Gitlab return [] if project && !project.repository.exists? if categories.any? - categories.keys.map do |category| + categories.keys.to_h do |category| files = self.by_category(category, project) [category, files.map { |t| { key: t.key, name: t.name, content: t.content } }] - end.to_h + end else files = self.all(project) files.map { |t| { key: t.key, name: t.name, content: t.content } } diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 9bb793a75cc..b16ae39bcee 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -4,35 +4,18 @@ module Gitlab module Tracking SNOWPLOW_NAMESPACE = 'gl' - module ControllerConcern - extend ActiveSupport::Concern - - protected - - def track_event(action = action_name, **args) - category = args.delete(:category) || self.class.name - Gitlab::Tracking.event(category, action.to_s, **args) - end - - def track_self_describing_event(schema_url, data:, **args) - Gitlab::Tracking.self_describing_event(schema_url, data: data, **args) - end - end - class << self def enabled? Gitlab::CurrentSettings.snowplow_enabled? end - def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil) # rubocop:disable Metrics/ParameterLists - contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace).to_context, *context] + def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists + contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace, **extra).to_context, *context] snowplow.event(category, action, label: label, property: property, value: value, context: contexts) product_analytics.event(category, action, label: label, property: property, value: value, context: contexts) - end - - def self_describing_event(schema_url, data:, context: nil) - snowplow.self_describing_event(schema_url, data: data, context: context) + rescue => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) end def snowplow_options(group) diff --git a/lib/gitlab/tracking/destinations/snowplow.rb b/lib/gitlab/tracking/destinations/snowplow.rb index 4fa844de325..e548532e061 100644 --- a/lib/gitlab/tracking/destinations/snowplow.rb +++ b/lib/gitlab/tracking/destinations/snowplow.rb @@ -15,13 +15,6 @@ module Gitlab tracker.track_struct_event(category, action, label, property, value, context, (Time.now.to_f * 1000).to_i) end - def self_describing_event(schema_url, data:, context: nil) - return unless enabled? - - event_json = SnowplowTracker::SelfDescribingJson.new(schema_url, data) - tracker.track_self_describing_event(event_json, context, (Time.now.to_f * 1000).to_i) - end - private def enabled? diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 8ce16c11267..da030649f76 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,11 +3,11 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-3'.freeze - GITLAB_RAILS_SOURCE = 'gitlab-rails'.freeze + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-4' + GITLAB_RAILS_SOURCE = 'gitlab-rails' - def initialize(namespace: nil, project: nil, user: nil, **data) - @data = data + def initialize(namespace: nil, project: nil, user: nil, **extra) + @extra = extra end def to_context @@ -35,8 +35,9 @@ module Gitlab def to_h { environment: environment, - source: source - }.merge(@data) + source: source, + extra: @extra + } end end end diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 6a3e2062070..706c0925302 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -35,6 +35,10 @@ module Gitlab matches end + def match(text) + scan_regexp.match(text) + end + def match?(text) text.present? && scan(text).present? end diff --git a/lib/gitlab/updated_notes_paginator.rb b/lib/gitlab/updated_notes_paginator.rb index 3d3d0e5bf9e..d5c01bde6b3 100644 --- a/lib/gitlab/updated_notes_paginator.rb +++ b/lib/gitlab/updated_notes_paginator.rb @@ -37,8 +37,8 @@ module Gitlab end def fetch_page(relation) - relation = relation.by_updated_at - notes = relation.at_most(LIMIT + 1).to_a + relation = relation.order_updated_asc.with_order_id_asc + notes = relation.limit(LIMIT + 1).to_a return [notes, false] unless notes.size > LIMIT diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb index 1dc660e574b..6b185a5a1e9 100644 --- a/lib/gitlab/usage/docs/helper.rb +++ b/lib/gitlab/usage/docs/helper.rb @@ -33,6 +33,10 @@ module Gitlab object[:description] end + def render_object_schema(object) + "[Object JSON schema](#{object.json_schema_path})" + end + def render_yaml_link(yaml_path) "[YAML definition](#{yaml_path})" end diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml index 19ad668019e..26f1aa4396d 100644 --- a/lib/gitlab/usage/docs/templates/default.md.haml +++ b/lib/gitlab/usage/docs/templates/default.md.haml @@ -27,6 +27,9 @@ = render_name(name) \ = render_description(object.attributes) + - if object.has_json_schema? + \ + = render_object_schema(object) \ = render_yaml_link(object.yaml_path) \ diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 4cb83348478..9c4255a7c92 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -5,6 +5,7 @@ module Gitlab class MetricDefinition METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' + SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze attr_reader :path attr_reader :attributes @@ -22,6 +23,16 @@ module Gitlab attributes end + def json_schema_path + return '' unless has_json_schema? + + "#{BASE_REPO_PATH}/#{attributes[:object_json_schema]}" + end + + def has_json_schema? + attributes[:value_type] == 'object' && attributes[:object_json_schema].present? + end + def yaml_path "#{BASE_REPO_PATH}#{path.delete_prefix(Rails.root.to_s)}" end @@ -29,7 +40,15 @@ module Gitlab def validate! unless skip_validation? self.class.schemer.validate(attributes.stringify_keys).each do |error| - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`")) + error_message = <<~ERROR_MSG + Error type: #{error['type']} + Data: #{error['data']} + Path: #{error['data_pointer']} + Details: #{error['details']} + Metric file: #{path} + ERROR_MSG + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Gitlab::Usage::Metric::InvalidMetricError.new(error_message)) end end end @@ -38,10 +57,11 @@ module Gitlab class << self def paths - @paths ||= [Rails.root.join('config', 'metrics', '**', '*.yml')] + @paths ||= [Rails.root.join('config', 'metrics', '[^agg]*', '*.yml')] end - def definitions + def definitions(skip_validation: false) + @skip_validation = skip_validation @definitions ||= load_all! end @@ -49,6 +69,10 @@ module Gitlab @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH)) end + def dump_metrics_yaml + @metrics_yaml ||= definitions.values.map(&:to_h).map(&:deep_stringify_keys).to_yaml + end + private def load_all! @@ -87,7 +111,7 @@ module Gitlab end def skip_validation? - !!attributes[:skip_validation] + !!attributes[:skip_validation] || @skip_validation || SKIP_VALIDATION_STATUSES.include?(attributes[:status]) end end end diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index 1aeca87d849..f77c8cab39c 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -7,7 +7,7 @@ module Gitlab UNION_OF_AGGREGATED_METRICS = 'OR' INTERSECTION_OF_AGGREGATED_METRICS = 'AND' ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze - AGGREGATED_METRICS_PATH = Rails.root.join('lib/gitlab/usage_data_counters/aggregated_metrics/*.yml') + AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml') AggregatedMetricError = Class.new(StandardError) UnknownAggregationOperator = Class.new(AggregatedMetricError) UnknownAggregationSource = Class.new(AggregatedMetricError) diff --git a/lib/gitlab/usage/metrics/names_suggestions/generator.rb b/lib/gitlab/usage/metrics/names_suggestions/generator.rb index 33f025770e0..49581169452 100644 --- a/lib/gitlab/usage/metrics/names_suggestions/generator.rb +++ b/lib/gitlab/usage/metrics/names_suggestions/generator.rb @@ -6,6 +6,8 @@ module Gitlab module NamesSuggestions class Generator < ::Gitlab::UsageData FREE_TEXT_METRIC_NAME = "<please fill metric name>" + REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>" + CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>" class << self def generate(key_path) @@ -23,7 +25,7 @@ module Gitlab end def redis_usage_counter - FREE_TEXT_METRIC_NAME + REDIS_EVENT_METRIC_NAME end def alt_usage_data(*) @@ -31,7 +33,7 @@ module Gitlab end def redis_usage_data_totals(counter) - counter.fallback_totals.transform_values { |_| FREE_TEXT_METRIC_NAME} + counter.fallback_totals.transform_values { |_| REDIS_EVENT_METRIC_NAME } end def sum(relation, column, *rest) @@ -47,49 +49,160 @@ module Gitlab end def name_suggestion(relation:, column: nil, prefix: nil, distinct: nil) - parts = [prefix] + # rubocop: disable CodeReuse/ActiveRecord + relation = relation.unscope(where: :created_at) + # rubocop: enable CodeReuse/ActiveRecord - if column - parts << parse_target(column) + parts = [prefix] + arel_column = arelize_column(relation, column) + + # nil as column indicates that the counting would use fallback value of primary key. + # Because counting primary key from relation is the conceptual equal to counting all + # records from given relation, in order to keep name suggestion more condensed + # primary key column is skipped. + # eg: SELECT COUNT(id) FROM issues would translate as count_issues and not + # as count_id_from_issues since it does not add more information to the name suggestion + if arel_column != Arel::Table.new(relation.table_name)[relation.primary_key] + parts << arel_column.name parts << 'from' end - source = parse_source(relation) - constraints = parse_constraints(relation: relation, column: column, distinct: distinct) + arel = arel_query(relation: relation, column: arel_column, distinct: distinct) + constraints = parse_constraints(relation: relation, arel: arel) + + # In some cases due to performance reasons metrics are instrumented with joined relations + # where relation listed in FROM statement is not the one that includes counted attribute + # in such situations to make name suggestion more intuitive source should be inferred based + # on the relation that provide counted attribute + # EG: SELECT COUNT(deployments.environment_id) FROM clusters + # JOIN deployments ON deployments.cluster_id = cluster.id + # should be translated into: + # count_environment_id_from_deployments_with_clusters + # instead of + # count_environment_id_from_clusters_with_deployments + actual_source = parse_source(relation, arel_column) + + append_constraints_prompt(actual_source, [constraints], parts) + + parts << actual_source + parts += process_joined_relations(actual_source, arel, relation, constraints) + parts.compact.join('_').delete('"') + end - if constraints.include?(source) - parts << "<adjective describing: '#{constraints}'>" - end + def append_constraints_prompt(target, constraints, parts) + applicable_constraints = constraints.select { |constraint| constraint.include?(target) } + return unless applicable_constraints.any? - parts << source - parts.compact.join('_') + parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') } end - def parse_constraints(relation:, column: nil, distinct: nil) + def parse_constraints(relation:, arel:) connection = relation.connection ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints .new(connection) - .accept(arel(relation: relation, column: column, distinct: distinct), collector(connection)) + .accept(arel, collector(connection)) .value end - def parse_target(column) - if column.is_a?(Arel::Attribute) - "#{column.relation.name}.#{column.name}" - else + # TODO: joins with `USING` keyword + def process_joined_relations(actual_source, arel, relation, where_constraints) + joins = parse_joins(connection: relation.connection, arel: arel) + return [] unless joins.any? + + sources = [relation.table_name, *joins.map { |join| join[:source] }] + joins = extract_joins_targets(joins, sources) + + relations = if actual_source != relation.table_name + build_relations_tree(joins + [{ source: relation.table_name }], actual_source) + else + # in case where counter attribute comes from joined relations, the relations + # diagram has to be built bottom up, thus source and target are reverted + build_relations_tree(joins + [{ source: relation.table_name }], actual_source, source_key: :target, target_key: :source) + end + + collect_join_parts(relations: relations[actual_source], joins: joins, wheres: where_constraints) + end + + def parse_joins(connection:, arel:) + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Joins + .new(connection) + .accept(arel) + end + + def extract_joins_targets(joins, sources) + joins.map do |join| + source_regex = /(#{join[:source]})\.(\w+_)*id/i + + tables_except_src = (sources - [join[:source]]).join('|') + target_regex = /(?<target>#{tables_except_src})\.(\w+_)*id/i + + join_cond_regex = /(#{source_regex}\s+=\s+#{target_regex})|(#{target_regex}\s+=\s+#{source_regex})/i + matched = join_cond_regex.match(join[:constraints]) + + if matched + join[:target] = matched[:target] + join[:constraints].gsub!(/#{join_cond_regex}(\s+(and|or))*/i, '') + end + + join + end + end + + def build_relations_tree(joins, parent, source_key: :source, target_key: :target) + return [] if joins.blank? + + tree = {} + tree[parent] = [] + + joins.each do |join| + if join[source_key] == parent + tree[parent] << build_relations_tree(joins - [join], join[target_key], source_key: source_key, target_key: target_key) + end + end + tree + end + + def collect_join_parts(relations:, joins:, wheres:, parts: [], conjunctions: %w[with having including].cycle) + conjunction = conjunctions.next + relations.each do |subtree| + subtree.each do |parent, children| + parts << "<#{conjunction}>" + join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints) + append_constraints_prompt(parent, [wheres, join_constraints].compact, parts) + parts << parent + collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions) + end + end + parts + end + + def arelize_column(relation, column) + case column + when Arel::Attribute column + when NilClass + Arel::Table.new(relation.table_name)[relation.primary_key] + when String + if column.include?('.') + table, col = column.split('.') + Arel::Table.new(table)[col] + else + Arel::Table.new(relation.table_name)[column] + end + when Symbol + arelize_column(relation, column.to_s) end end - def parse_source(relation) - relation.table_name + def parse_source(relation, column) + column.relation.name || relation.table_name end def collector(connection) Arel::Collectors::SubstituteBinds.new(connection, Arel::Collectors::SQLString.new) end - def arel(relation:, column: nil, distinct: nil) + def arel_query(relation:, column: nil, distinct: nil) column ||= relation.primary_key if column.is_a?(Arel::Attribute) diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb new file mode 100644 index 00000000000..d52e4903f3c --- /dev/null +++ b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/joins.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module NamesSuggestions + module RelationParsers + class Joins < ::Arel::Visitors::PostgreSQL + def accept(object) + object.source.right.map do |join| + visit(join, collector) + end + end + + private + + # rubocop:disable Naming/MethodName + def visit_Arel_Nodes_StringJoin(object, collector) + result = visit(object.left, collector) + source, constraints = result.value.split('ON') + { + source: source.split('JOIN').last&.strip, + constraints: constraints&.strip + }.compact + end + + def visit_Arel_Nodes_FullOuterJoin(object, _) + parse_join(object) + end + + def visit_Arel_Nodes_OuterJoin(object, _) + parse_join(object) + end + + def visit_Arel_Nodes_RightOuterJoin(object, _) + parse_join(object) + end + + def visit_Arel_Nodes_InnerJoin(object, _) + { + source: visit(object.left, collector).value, + constraints: object.right ? visit(object.right.expr, collector).value : nil + }.compact + end + # rubocop:enable Naming/MethodName + + def parse_join(object) + { + source: visit(object.left, collector).value, + constraints: visit(object.right.expr, collector).value + } + end + + def quote(value) + "#{value}" + end + + def quote_table_name(name) + "#{name}" + end + + def quote_column_name(name) + "#{name}" + end + + def collector + Arel::Collectors::SubstituteBinds.new(@connection, Arel::Collectors::SQLString.new) + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 5dc3f71329d..b36ca38cd64 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -87,7 +87,7 @@ module Gitlab # rubocop: disable Metrics/AbcSize # rubocop: disable CodeReuse/ActiveRecord def system_usage_data - issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id) + issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)) { counts: { @@ -138,7 +138,7 @@ module Gitlab in_review_folder: count(::Environment.in_review_folder), grafana_integrated_projects: count(GrafanaIntegration.enabled), groups: count(Group), - issues: count(Issue, start: issue_minimum_id, finish: issue_maximum_id), + issues: count(Issue, start: minimum_id(Issue), finish: maximum_id(Issue)), issues_created_from_gitlab_error_tracking_ui: count(SentryIssue), issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue), issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), @@ -146,9 +146,9 @@ module Gitlab issues_created_from_alerts: total_alert_issues, issues_created_gitlab_alerts: issues_created_manually_from_alerts, issues_created_manually_from_alerts: issues_created_manually_from_alerts, - incident_issues: count(::Issue.incident, start: issue_minimum_id, finish: issue_maximum_id), - alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: issue_minimum_id, finish: issue_maximum_id), - incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: issue_minimum_id, finish: issue_maximum_id), + incident_issues: count(::Issue.incident, start: minimum_id(Issue), finish: maximum_id(Issue)), + alert_bot_incident_issues: count(::Issue.authored(::User.alert_bot), start: minimum_id(Issue), finish: maximum_id(Issue)), + incident_labeled_issues: count(::Issue.with_label_attributes(::IncidentManagement::CreateIncidentLabelService::LABEL_PROPERTIES), start: minimum_id(Issue), finish: maximum_id(Issue)), keys: count(Key), label_lists: count(List.label), lfs_objects: count(LfsObject), @@ -389,8 +389,8 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def container_expiration_policies_usage results = {} - start = ::Project.minimum(:id) - finish = ::Project.maximum(:id) + start = minimum_id(Project) + finish = maximum_id(Project) results[:projects_with_expiration_policy_disabled] = distinct_count(::ContainerExpirationPolicy.where(enabled: false), :project_id, start: start, finish: finish) # rubocop: disable UsageData/LargeTable @@ -591,7 +591,7 @@ module Gitlab { events: distinct_count(::Event.where(time_period), :author_id), groups: distinct_count(::GroupMember.where(time_period), :user_id), - users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id), + users_created: count(::User.where(time_period), start: minimum_id(User), finish: maximum_id(User)), omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), unique_users_all_imports: unique_users_all_imports(time_period), @@ -636,8 +636,8 @@ module Gitlab clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id), clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period), operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period), - start: user_minimum_id, - finish: user_maximum_id), + start: minimum_id(User), + finish: maximum_id(User)), projects_with_tracing_enabled: distinct_count(::Project.with_tracing_enabled.where(time_period), :creator_id), projects_with_error_tracking_enabled: distinct_count(::Project.with_enabled_error_tracking.where(time_period), :creator_id), projects_with_incidents: distinct_count(::Issue.incident.where(time_period), :project_id), @@ -691,12 +691,12 @@ module Gitlab def usage_activity_by_stage_verify(time_period) { ci_builds: distinct_count(::Ci::Build.where(time_period), :user_id), - ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), - ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), - ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), - ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), + ci_external_pipelines: distinct_count(::Ci::Pipeline.external.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)), + ci_internal_pipelines: distinct_count(::Ci::Pipeline.internal.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)), + ci_pipeline_config_auto_devops: distinct_count(::Ci::Pipeline.auto_devops_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)), + ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)), ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id), - ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: user_minimum_id, finish: user_maximum_id), + ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)), ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id), clusters_applications_runner: cluster_applications_user_distinct_count(::Clusters::Applications::Runner, time_period) } @@ -711,6 +711,8 @@ module Gitlab end def redis_hll_counters + return {} unless Feature.enabled?(:redis_hll_tracking, type: :ops, default_enabled: :yaml) + { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } end @@ -799,8 +801,8 @@ module Gitlab end def distinct_count_service_desk_enabled_projects(time_period) - project_creator_id_start = user_minimum_id - project_creator_id_finish = user_maximum_id + project_creator_id_start = minimum_id(User) + project_creator_id_finish = maximum_id(User) distinct_count(::Project.service_desk_enabled.where(time_period), :creator_id, start: project_creator_id_start, finish: project_creator_id_finish) # rubocop: disable CodeReuse/ActiveRecord end @@ -832,57 +834,9 @@ module Gitlab def total_alert_issues # Remove prometheus table queries once they are deprecated # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407. - add count(Issue.with_alert_management_alerts, start: issue_minimum_id, finish: issue_maximum_id), - count(::Issue.with_self_managed_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id), - count(::Issue.with_prometheus_alert_events, start: issue_minimum_id, finish: issue_maximum_id) - end - - def user_minimum_id - strong_memoize(:user_minimum_id) do - ::User.minimum(:id) - end - end - - def user_maximum_id - strong_memoize(:user_maximum_id) do - ::User.maximum(:id) - end - end - - def issue_minimum_id - strong_memoize(:issue_minimum_id) do - ::Issue.minimum(:id) - end - end - - def issue_maximum_id - strong_memoize(:issue_maximum_id) do - ::Issue.maximum(:id) - end - end - - def deployment_minimum_id - strong_memoize(:deployment_minimum_id) do - ::Deployment.minimum(:id) - end - end - - def deployment_maximum_id - strong_memoize(:deployment_maximum_id) do - ::Deployment.maximum(:id) - end - end - - def project_minimum_id - strong_memoize(:project_minimum_id) do - ::Project.minimum(:id) - end - end - - def project_maximum_id - strong_memoize(:project_maximum_id) do - ::Project.maximum(:id) - end + add count(Issue.with_alert_management_alerts, start: minimum_id(Issue), finish: maximum_id(Issue)), + count(::Issue.with_self_managed_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue)), + count(::Issue.with_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue)) end def self_monitoring_project @@ -916,7 +870,7 @@ module Gitlab end def deployment_count(relation) - count relation, start: deployment_minimum_id, finish: deployment_maximum_id + count relation, start: minimum_id(Deployment), finish: maximum_id(Deployment) end def project_imports(time_period) diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml deleted file mode 100644 index 4c2355d526a..00000000000 --- a/lib/gitlab/usage_data_counters/aggregated_metrics/code_review.yml +++ /dev/null @@ -1,108 +0,0 @@ -# code_review_extension_category_monthly_active_users -# This is only metrics related to the VS Code Extension for now. -# -# code_review_category_monthly_active_users -# This is the user based metrics. These should only be user based metrics and only be related to the Code Review things inside of GitLab. -# -# code_review_group_monthly_active_users -# This is an aggregation of both of the above aggregations. It's intended to represent all users who interact with our group across all of our categories. ---- -- name: code_review_group_monthly_active_users - operator: OR - feature_flag: usage_data_code_review_aggregation - source: redis - time_frame: [7d, 28d] - events: [ - 'i_code_review_user_single_file_diffs', - 'i_code_review_user_create_mr', - 'i_code_review_user_close_mr', - 'i_code_review_user_reopen_mr', - 'i_code_review_user_resolve_thread', - 'i_code_review_user_unresolve_thread', - 'i_code_review_edit_mr_title', - 'i_code_review_edit_mr_desc', - 'i_code_review_user_merge_mr', - 'i_code_review_user_create_mr_comment', - 'i_code_review_user_edit_mr_comment', - 'i_code_review_user_remove_mr_comment', - 'i_code_review_user_create_review_note', - 'i_code_review_user_publish_review', - 'i_code_review_user_create_multiline_mr_comment', - 'i_code_review_user_edit_multiline_mr_comment', - 'i_code_review_user_remove_multiline_mr_comment', - 'i_code_review_user_add_suggestion', - 'i_code_review_user_apply_suggestion', - 'i_code_review_user_assigned', - 'i_code_review_user_review_requested', - 'i_code_review_user_approve_mr', - 'i_code_review_user_unapprove_mr', - 'i_code_review_user_marked_as_draft', - 'i_code_review_user_unmarked_as_draft', - 'i_code_review_user_approval_rule_added', - 'i_code_review_user_approval_rule_deleted', - 'i_code_review_user_approval_rule_edited', - 'i_code_review_user_vs_code_api_request', - 'i_code_review_user_toggled_task_item_status', - 'i_code_review_user_create_mr_from_issue', - 'i_code_review_user_mr_discussion_locked', - 'i_code_review_user_mr_discussion_unlocked', - 'i_code_review_user_time_estimate_changed', - 'i_code_review_user_time_spent_changed', - 'i_code_review_user_assignees_changed', - 'i_code_review_user_reviewers_changed', - 'i_code_review_user_milestone_changed', - 'i_code_review_user_labels_changed' - ] -- name: code_review_category_monthly_active_users - operator: OR - feature_flag: usage_data_code_review_aggregation - source: redis - time_frame: [7d, 28d] - events: [ - 'i_code_review_user_single_file_diffs', - 'i_code_review_user_create_mr', - 'i_code_review_user_close_mr', - 'i_code_review_user_reopen_mr', - 'i_code_review_user_resolve_thread', - 'i_code_review_user_unresolve_thread', - 'i_code_review_edit_mr_title', - 'i_code_review_edit_mr_desc', - 'i_code_review_user_merge_mr', - 'i_code_review_user_create_mr_comment', - 'i_code_review_user_edit_mr_comment', - 'i_code_review_user_remove_mr_comment', - 'i_code_review_user_create_review_note', - 'i_code_review_user_publish_review', - 'i_code_review_user_create_multiline_mr_comment', - 'i_code_review_user_edit_multiline_mr_comment', - 'i_code_review_user_remove_multiline_mr_comment', - 'i_code_review_user_add_suggestion', - 'i_code_review_user_apply_suggestion', - 'i_code_review_user_assigned', - 'i_code_review_user_review_requested', - 'i_code_review_user_approve_mr', - 'i_code_review_user_unapprove_mr', - 'i_code_review_user_marked_as_draft', - 'i_code_review_user_unmarked_as_draft', - 'i_code_review_user_approval_rule_added', - 'i_code_review_user_approval_rule_deleted', - 'i_code_review_user_approval_rule_edited', - 'i_code_review_user_toggled_task_item_status', - 'i_code_review_user_create_mr_from_issue', - 'i_code_review_user_mr_discussion_locked', - 'i_code_review_user_mr_discussion_unlocked', - 'i_code_review_user_time_estimate_changed', - 'i_code_review_user_time_spent_changed', - 'i_code_review_user_assignees_changed', - 'i_code_review_user_reviewers_changed', - 'i_code_review_user_milestone_changed', - 'i_code_review_user_labels_changed' - ] -- name: code_review_extension_category_monthly_active_users - operator: OR - feature_flag: usage_data_code_review_aggregation - source: redis - time_frame: [7d, 28d] - events: [ - 'i_code_review_user_vs_code_api_request' - ] diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml deleted file mode 100644 index 73a55b5d5fa..00000000000 --- a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml +++ /dev/null @@ -1,72 +0,0 @@ -# Aggregated metrics that include EE only event names within `events:` attribute have to be defined at ee/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml -# instead of this file. -#- name: unique name of aggregated metric -# operator: aggregation operator. Valid values are: -# - "OR": counts unique elements that were observed triggering any of following events -# - "AND": counts unique elements that were observed triggering all of following events -# events: list of events names to aggregate into metric. All events in this list must have the same 'redis_slot' and 'aggregation' attributes -# see from lib/gitlab/usage_data_counters/known_events/ for the list of valid events. -# source: defines which datasource will be used to locate events that should be included in aggregated metric. Valid values are: -# - database -# - redis -# time_frame: defines time frames for aggregated metrics: -# - 7d - last 7 days -# - 28d - last 28 days -# - all - all historical available data, this time frame is not available for redis source -# feature_flag: name of development feature flag that will be checked before metrics aggregation is performed. -# Corresponding feature flag should have `default_enabled` attribute set to `false`. -# This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked. ---- -- name: compliance_features_track_unique_visits_union - operator: OR - source: redis - time_frame: [7d, 28d] - events: ['g_compliance_audit_events', 'g_compliance_dashboard', 'i_compliance_audit_events', 'a_compliance_audit_events_api', 'i_compliance_credential_inventory'] -- name: product_analytics_test_metrics_union - operator: OR - source: redis - time_frame: [7d, 28d] - events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] -- name: product_analytics_test_metrics_intersection - operator: AND - source: redis - time_frame: [7d, 28d] - events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] -- name: incident_management_alerts_total_unique_counts - operator: OR - source: redis - time_frame: [7d, 28d] - events: [ - 'incident_management_alert_status_changed', - 'incident_management_alert_assigned', - 'incident_management_alert_todo', - 'incident_management_alert_create_incident' - ] -- name: incident_management_incidents_total_unique_counts - operator: OR - source: redis - time_frame: [7d, 28d] - events: [ - 'incident_management_incident_created', - 'incident_management_incident_reopened', - 'incident_management_incident_closed', - 'incident_management_incident_assigned', - 'incident_management_incident_todo', - 'incident_management_incident_comment', - 'incident_management_incident_zoom_meeting', - 'incident_management_incident_published', - 'incident_management_incident_relate', - 'incident_management_incident_unrelate', - 'incident_management_incident_change_confidential' - ] -- name: i_testing_paid_monthly_active_user_total - operator: OR - source: redis - time_frame: [7d, 28d] - events: [ - 'i_testing_web_performance_widget_total', - 'i_testing_full_code_quality_report_total', - 'i_testing_group_code_coverage_visit_total', - 'i_testing_load_performance_widget_total', - 'i_testing_metrics_report_widget_total' - ] diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb index d28fd17a989..4ab310a2519 100644 --- a/lib/gitlab/usage_data_counters/base_counter.rb +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -22,11 +22,11 @@ module Gitlab::UsageDataCounters end def totals - known_events.map { |event| [counter_key(event), read(event)] }.to_h + known_events.to_h { |event| [counter_key(event), read(event)] } end def fallback_totals - known_events.map { |event| [counter_key(event), -1] }.to_h + known_events.to_h { |event| [counter_key(event), -1] } end def fetch_supported_event(event_name) diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index 772a4623280..c9106d7c6b8 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -2,7 +2,7 @@ module Gitlab::UsageDataCounters class CiTemplateUniqueCounter - REDIS_SLOT = 'ci_templates'.freeze + REDIS_SLOT = 'ci_templates' # NOTE: Events originating from implicit Auto DevOps pipelines get prefixed with `implicit_` TEMPLATE_TO_EVENT = { @@ -20,8 +20,6 @@ module Gitlab::UsageDataCounters class << self def track_unique_project_event(project_id:, template:, config_source:) - return if Feature.disabled?(:usage_data_track_ci_templates_unique_projects, default_enabled: :yaml) - if event = unique_project_event(template, config_source) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id) end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 336bef081a6..a8691169fb8 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -151,13 +151,16 @@ module Gitlab aggregation = events.first[:aggregation] keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context) + + return FALLBACK unless keys.any? + redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end def feature_enabled?(event) return true if event[:feature_flag].blank? - Feature.enabled?(event[:feature_flag], default_enabled: :yaml) + Feature.enabled?(event[:feature_flag], default_enabled: :yaml) && Feature.enabled?(:redis_hll_tracking, type: :ops, default_enabled: :yaml) end # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index c2662a74432..6f5f878501f 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -34,120 +34,120 @@ module Gitlab ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed' class << self - def track_issue_created_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_CREATED, author, time) + def track_issue_created_action(author:) + track_unique_action(ISSUE_CREATED, author) end - def track_issue_title_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_TITLE_CHANGED, author, time) + def track_issue_title_changed_action(author:) + track_unique_action(ISSUE_TITLE_CHANGED, author) end - def track_issue_description_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_DESCRIPTION_CHANGED, author, time) + def track_issue_description_changed_action(author:) + track_unique_action(ISSUE_DESCRIPTION_CHANGED, author) end - def track_issue_assignee_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_ASSIGNEE_CHANGED, author, time) + def track_issue_assignee_changed_action(author:) + track_unique_action(ISSUE_ASSIGNEE_CHANGED, author) end - def track_issue_made_confidential_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_MADE_CONFIDENTIAL, author, time) + def track_issue_made_confidential_action(author:) + track_unique_action(ISSUE_MADE_CONFIDENTIAL, author) end - def track_issue_made_visible_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_MADE_VISIBLE, author, time) + def track_issue_made_visible_action(author:) + track_unique_action(ISSUE_MADE_VISIBLE, author) end - def track_issue_closed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_CLOSED, author, time) + def track_issue_closed_action(author:) + track_unique_action(ISSUE_CLOSED, author) end - def track_issue_reopened_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_REOPENED, author, time) + def track_issue_reopened_action(author:) + track_unique_action(ISSUE_REOPENED, author) end - def track_issue_label_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_LABEL_CHANGED, author, time) + def track_issue_label_changed_action(author:) + track_unique_action(ISSUE_LABEL_CHANGED, author) end - def track_issue_milestone_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_MILESTONE_CHANGED, author, time) + def track_issue_milestone_changed_action(author:) + track_unique_action(ISSUE_MILESTONE_CHANGED, author) end - def track_issue_cross_referenced_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_CROSS_REFERENCED, author, time) + def track_issue_cross_referenced_action(author:) + track_unique_action(ISSUE_CROSS_REFERENCED, author) end - def track_issue_moved_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_MOVED, author, time) + def track_issue_moved_action(author:) + track_unique_action(ISSUE_MOVED, author) end - def track_issue_related_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_RELATED, author, time) + def track_issue_related_action(author:) + track_unique_action(ISSUE_RELATED, author) end - def track_issue_unrelated_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_UNRELATED, author, time) + def track_issue_unrelated_action(author:) + track_unique_action(ISSUE_UNRELATED, author) end - def track_issue_marked_as_duplicate_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author, time) + def track_issue_marked_as_duplicate_action(author:) + track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author) end - def track_issue_locked_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_LOCKED, author, time) + def track_issue_locked_action(author:) + track_unique_action(ISSUE_LOCKED, author) end - def track_issue_unlocked_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_UNLOCKED, author, time) + def track_issue_unlocked_action(author:) + track_unique_action(ISSUE_UNLOCKED, author) end - def track_issue_designs_added_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_DESIGNS_ADDED, author, time) + def track_issue_designs_added_action(author:) + track_unique_action(ISSUE_DESIGNS_ADDED, author) end - def track_issue_designs_modified_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_DESIGNS_MODIFIED, author, time) + def track_issue_designs_modified_action(author:) + track_unique_action(ISSUE_DESIGNS_MODIFIED, author) end - def track_issue_designs_removed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_DESIGNS_REMOVED, author, time) + def track_issue_designs_removed_action(author:) + track_unique_action(ISSUE_DESIGNS_REMOVED, author) end - def track_issue_due_date_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_DUE_DATE_CHANGED, author, time) + def track_issue_due_date_changed_action(author:) + track_unique_action(ISSUE_DUE_DATE_CHANGED, author) end - def track_issue_time_estimate_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author, time) + def track_issue_time_estimate_changed_action(author:) + track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author) end - def track_issue_time_spent_changed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_TIME_SPENT_CHANGED, author, time) + def track_issue_time_spent_changed_action(author:) + track_unique_action(ISSUE_TIME_SPENT_CHANGED, author) end - def track_issue_comment_added_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_COMMENT_ADDED, author, time) + def track_issue_comment_added_action(author:) + track_unique_action(ISSUE_COMMENT_ADDED, author) end - def track_issue_comment_edited_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_COMMENT_EDITED, author, time) + def track_issue_comment_edited_action(author:) + track_unique_action(ISSUE_COMMENT_EDITED, author) end - def track_issue_comment_removed_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_COMMENT_REMOVED, author, time) + def track_issue_comment_removed_action(author:) + track_unique_action(ISSUE_COMMENT_REMOVED, author) end - def track_issue_cloned_action(author:, time: Time.zone.now) - track_unique_action(ISSUE_CLONED, author, time) + def track_issue_cloned_action(author:) + track_unique_action(ISSUE_CLONED, author) end private - def track_unique_action(action, author, time) + def track_unique_action(action, author) return unless author - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id) end end end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index 9c19c9e8b8c..3c692f2b1af 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -3,89 +3,74 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_implicit_auto_devops_build category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_implicit_auto_devops_deploy category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_implicit_security_sast category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_implicit_security_secret_detection category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects # Explicit include:template pipeline events - name: p_ci_templates_5_min_production_app category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_auto_devops category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_aws_cf_deploy_ec2 category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_aws_deploy_ecs category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_auto_devops_build category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_auto_devops_deploy category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_auto_devops_deploy_latest category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_security_sast category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_security_secret_detection category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects - name: p_ci_templates_terraform_base_latest category: ci_templates redis_slot: ci_templates aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 80a79682338..077864032e8 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -91,6 +91,11 @@ redis_slot: analytics aggregation: weekly feature_flag: track_unique_visits +- name: i_analytics_dev_ops_adoption + category: analytics + redis_slot: analytics + aggregation: weekly + feature_flag: track_unique_visits - name: g_analytics_merge_request category: analytics redis_slot: analytics @@ -242,6 +247,12 @@ category: incident_management_alerts aggregation: weekly feature_flag: usage_data_incident_management_alert_create_incident +# Incident management on-call +- name: i_incident_management_oncall_notification_sent + redis_slot: incident_management + category: incident_management_oncall + aggregation: weekly + feature_flag: usage_data_i_incident_management_oncall_notification_sent # Testing category - name: i_testing_test_case_parsed category: testing @@ -283,6 +294,11 @@ redis_slot: testing aggregation: weekly feature_flag: usage_data_i_testing_metrics_report_artifact_uploaders +- name: i_testing_summary_widget_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_summary_widget_total # Project Management group - name: g_project_management_issue_title_changed category: issues_edit @@ -444,13 +460,19 @@ redis_slot: pipeline_authoring aggregation: weekly feature_flag: usage_data_o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile -# Epic events -# -# We are using the same slot of issue events 'project_management' for -# epic events to allow data aggregation. -# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 -- name: g_project_management_epic_created - category: epics_usage - redis_slot: project_management - aggregation: daily - feature_flag: track_epics_activity +# Merge request widgets +- name: users_expanding_secure_security_report + redis_slot: secure + category: secure + aggregation: weekly + feature_flag: users_expanding_widgets_usage_data +- name: users_expanding_testing_code_quality_report + redis_slot: testing + category: testing + aggregation: weekly + feature_flag: users_expanding_widgets_usage_data +- name: users_expanding_testing_accessibility_report + redis_slot: testing + category: testing + aggregation: weekly + feature_flag: users_expanding_widgets_usage_data diff --git a/lib/gitlab/usage_data_counters/known_events/epic_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_events.yml new file mode 100644 index 00000000000..80460dbe4d2 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/epic_events.yml @@ -0,0 +1,142 @@ +# Epic events +# +# We are using the same slot of issue events 'project_management' for +# epic events to allow data aggregation. +# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 +- name: g_project_management_epic_created + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_updating_epic_titles + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_updating_epic_descriptions + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +# epic notes + +- name: g_project_management_users_creating_epic_notes + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_updating_epic_notes + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_destroying_epic_notes + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +# start date events + +- name: g_project_management_users_setting_epic_start_date_as_fixed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_updating_fixed_epic_start_date + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_setting_epic_start_date_as_inherited + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +# due date events + +- name: g_project_management_users_setting_epic_due_date_as_fixed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_updating_fixed_epic_due_date + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_setting_epic_due_date_as_inherited + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_issue_added + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_issue_removed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_issue_moved_from_project + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_closed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_reopened + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: 'g_project_management_issue_promoted_to_epic' + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_setting_epic_confidential + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_users_setting_epic_visible + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_users_changing_labels + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity + +- name: g_project_management_epic_destroyed + category: epics_usage + redis_slot: project_management + aggregation: daily + feature_flag: track_epics_activity diff --git a/lib/gitlab/usage_data_counters/note_counter.rb b/lib/gitlab/usage_data_counters/note_counter.rb index 7a76180cb08..aae2d144c5b 100644 --- a/lib/gitlab/usage_data_counters/note_counter.rb +++ b/lib/gitlab/usage_data_counters/note_counter.rb @@ -24,13 +24,13 @@ module Gitlab::UsageDataCounters end def totals - COUNTABLE_TYPES.map do |countable_type| + COUNTABLE_TYPES.to_h do |countable_type| [counter_key(countable_type), read(:create, countable_type)] - end.to_h + end end def fallback_totals - COUNTABLE_TYPES.map { |counter_key| [counter_key(counter_key), -1] }.to_h + COUNTABLE_TYPES.to_h { |counter_key| [counter_key(counter_key), -1] } end private diff --git a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb index 15c68fb3945..ed3df7dcf75 100644 --- a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb @@ -28,7 +28,7 @@ module Gitlab 'unassign_reviewer' when 'request_review', 'reviewer' 'assign_reviewer' - when 'spend' + when 'spend', 'spent' event_name_for_spend(args) when 'unassign' event_name_for_unassign(args) diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb new file mode 100644 index 00000000000..1f72bf4ce26 --- /dev/null +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + class UsageDataNonSqlMetrics < UsageData + SQL_METRIC_DEFAULT = -3 + + class << self + def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) + SQL_METRIC_DEFAULT + end + + def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) + SQL_METRIC_DEFAULT + end + + def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) + SQL_METRIC_DEFAULT + end + + def sum(relation, column, batch_size: nil, start: nil, finish: nil) + SQL_METRIC_DEFAULT + end + + def histogram(relation, column, buckets:, bucket_size: buckets.size) + SQL_METRIC_DEFAULT + end + + def maximum_id(model) + end + + def minimum_id(model) + end + end + end +end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index c00e7a2aa13..c0dfae88fc7 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -5,11 +5,11 @@ module Gitlab # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41091 class UsageDataQueries < UsageData class << self - def count(relation, column = nil, *rest) + def count(relation, column = nil, *args, **kwargs) raw_sql(relation, column) end - def distinct_count(relation, column = nil, *rest) + def distinct_count(relation, column = nil, *args, **kwargs) raw_sql(relation, column, :distinct) end @@ -21,14 +21,14 @@ module Gitlab end end - def sum(relation, column, *rest) + def sum(relation, column, *args, **kwargs) relation.select(relation.all.table[column].sum).to_sql end # For estimated distinct count use exact query instead of hll # buckets query, because it can't be used to obtain estimations without # supplementary ruby code present in Gitlab::Database::PostgresHll::BatchDistinctCounter - def estimate_batch_distinct_count(relation, column = nil, *rest) + def estimate_batch_distinct_count(relation, column = nil, *args, **kwargs) raw_sql(relation, column, :distinct) end @@ -36,6 +36,12 @@ module Gitlab 'SELECT ' + args.map {|arg| "(#{arg})" }.join(' + ') end + def maximum_id(model) + end + + def minimum_id(model) + end + private def raw_sql(relation, column, distinct = nil) diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 29f02a5912a..c1a57566640 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -99,6 +99,8 @@ module Gitlab end def to_boolean(value, default: nil) + value = value.to_s if [0, 1].include?(value) + return value if [true, false].include?(value) return true if value =~ /^(true|t|yes|y|1|on)$/i return false if value =~ /^(false|f|no|n|0|off)$/i diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 854fc5c917d..efa2f7a943f 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -36,6 +36,7 @@ module Gitlab module Utils module UsageData + include Gitlab::Utils::StrongMemoize extend self FALLBACK = -1 @@ -209,6 +210,20 @@ module Gitlab Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values) end + def maximum_id(model) + key = :"#{model.name.downcase}_maximum_id" + strong_memoize(key) do + model.maximum(:id) + end + end + + def minimum_id(model) + key = :"#{model.name.downcase}_minimum_id" + strong_memoize(key) do + model.minimum(:id) + end + end + private def prometheus_client(verify:) diff --git a/lib/gitlab/uuid.rb b/lib/gitlab/uuid.rb index 80caf2c6788..016c25eb94b 100644 --- a/lib/gitlab/uuid.rb +++ b/lib/gitlab/uuid.rb @@ -9,9 +9,9 @@ module Gitlab production: "58dc0f06-936c-43b3-93bb-71693f1b6570" }.freeze - UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{4}\h{8}/.freeze + UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{12}/.freeze NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze - PACK_PATTERN = "NnnnnN".freeze + PACK_PATTERN = "NnnnnN" class << self def v5(name, namespace_id: default_namespace_id) diff --git a/lib/gitlab/web_ide/config/entry/terminal.rb b/lib/gitlab/web_ide/config/entry/terminal.rb index 403e308d45b..ec07023461f 100644 --- a/lib/gitlab/web_ide/config/entry/terminal.rb +++ b/lib/gitlab/web_ide/config/entry/terminal.rb @@ -10,6 +10,7 @@ module Gitlab class Terminal < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable + include Gitlab::Utils::StrongMemoize # By default the build will finish in a few seconds, not giving the webide # enough time to connect to the terminal. This default script provides @@ -51,21 +52,26 @@ module Gitlab private def to_hash - { tag_list: tags || [], - yaml_variables: yaml_variables, + { + tag_list: tags || [], + yaml_variables: yaml_variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + job_variables: yaml_variables, options: { image: image_value, services: services_value, before_script: before_script_value, script: script_value || DEFAULT_SCRIPT - }.compact } + }.compact + }.compact end def yaml_variables - return unless variables_value + strong_memoize(:yaml_variables) do + next unless variables_value - variables_value.map do |key, value| - { key: key.to_s, value: value, public: true } + variables_value.map do |key, value| + { key: key.to_s, value: value, public: true } + end end end end diff --git a/lib/gitlab/word_diff/chunk_collection.rb b/lib/gitlab/word_diff/chunk_collection.rb index dd388f75302..d5c3e59d405 100644 --- a/lib/gitlab/word_diff/chunk_collection.rb +++ b/lib/gitlab/word_diff/chunk_collection.rb @@ -18,6 +18,27 @@ module Gitlab def reset @chunks = [] end + + def marker_ranges + start = 0 + + @chunks.each_with_object([]) do |element, ranges| + mode = mode_for_element(element) + + ranges << Gitlab::MarkerRange.new(start, start + element.length - 1, mode: mode) if mode + + start += element.length + end + end + + private + + def mode_for_element(element) + return Gitlab::MarkerRange::DELETION if element.removed? + return Gitlab::MarkerRange::ADDITION if element.added? + + nil + end end end end diff --git a/lib/gitlab/word_diff/parser.rb b/lib/gitlab/word_diff/parser.rb index 3b6d4d4d384..e611abb5692 100644 --- a/lib/gitlab/word_diff/parser.rb +++ b/lib/gitlab/word_diff/parser.rb @@ -31,7 +31,7 @@ module Gitlab @chunks.add(segment) when Segments::Newline - yielder << build_line(@chunks.content, nil, parent_file: diff_file) + yielder << build_line(@chunks.content, nil, parent_file: diff_file).tap { |line| line.set_marker_ranges(@chunks.marker_ranges) } @chunks.reset counter.increase_pos_num |