diff options
Diffstat (limited to 'lib/gitlab')
221 files changed, 4052 insertions, 973 deletions
diff --git a/lib/gitlab/alert_management/alert_params.rb b/lib/gitlab/alert_management/alert_params.rb index 982479784a9..789a4fe246a 100644 --- a/lib/gitlab/alert_management/alert_params.rb +++ b/lib/gitlab/alert_management/alert_params.rb @@ -20,7 +20,8 @@ module Gitlab hosts: Array(annotations[:hosts]), payload: payload, started_at: parsed_payload['startsAt'], - severity: annotations[:severity] + severity: annotations[:severity], + fingerprint: annotations[:fingerprint] } end diff --git a/lib/gitlab/alert_management/fingerprint.rb b/lib/gitlab/alert_management/fingerprint.rb new file mode 100644 index 00000000000..6ab47c88ca1 --- /dev/null +++ b/lib/gitlab/alert_management/fingerprint.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module AlertManagement + class Fingerprint + def self.generate(data) + new.generate(data) + end + + def generate(data) + return unless data.present? + + if data.is_a?(Array) + data = flatten_array(data) + end + + Digest::SHA1.hexdigest(data.to_s) + end + + private + + def flatten_array(array) + array.flatten.map!(&:to_s).join + end + end + end +end diff --git a/lib/gitlab/alerting/alert.rb b/lib/gitlab/alerting/alert.rb index d859ca89418..dad3dabb4fc 100644 --- a/lib/gitlab/alerting/alert.rb +++ b/lib/gitlab/alerting/alert.rb @@ -106,7 +106,7 @@ module Gitlab end def gitlab_fingerprint - Digest::SHA1.hexdigest(plain_gitlab_fingerprint) + Gitlab::AlertManagement::Fingerprint.generate(plain_gitlab_fingerprint) end def valid? @@ -121,9 +121,9 @@ module Gitlab def plain_gitlab_fingerprint if gitlab_managed? - [metric_id, starts_at].join('/') + [metric_id, starts_at_raw].join('/') else # self managed - [starts_at, title, full_query].join('/') + [starts_at_raw, title, full_query].join('/') end end @@ -173,7 +173,10 @@ module Gitlab value = payload&.dig(field) return unless value - Time.rfc3339(value) + # value is a rfc3339 timestamp + # Timestamps from Prometheus and Alertmanager are UTC RFC3339 timestamps like: '2018-03-12T09:06:00Z' (Z represents 0 offset or UTC) + # .utc sets the datetime zone to `UTC` + Time.rfc3339(value).utc rescue ArgumentError end diff --git a/lib/gitlab/alerting/notification_payload_parser.rb b/lib/gitlab/alerting/notification_payload_parser.rb index c79d69613f3..d98b9296347 100644 --- a/lib/gitlab/alerting/notification_payload_parser.rb +++ b/lib/gitlab/alerting/notification_payload_parser.rb @@ -35,6 +35,10 @@ module Gitlab payload[:severity].presence || DEFAULT_SEVERITY end + def fingerprint + Gitlab::AlertManagement::Fingerprint.generate(payload[:fingerprint]) + end + def annotations primary_params .reverse_merge(flatten_secondary_params) @@ -49,7 +53,8 @@ module Gitlab 'monitoring_tool' => payload[:monitoring_tool], 'service' => payload[:service], 'hosts' => hosts.presence, - 'severity' => severity + 'severity' => severity, + 'fingerprint' => fingerprint } end diff --git a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb index 9ea20a4d6a4..4dec71b35e8 100644 --- a/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb +++ b/lib/gitlab/analytics/cycle_analytics/base_query_builder.rb @@ -8,21 +8,24 @@ module Gitlab delegate :subject_class, to: :stage - # rubocop: disable CodeReuse/ActiveRecord + FINDER_CLASSES = { + MergeRequest.to_s => MergeRequestsFinder, + Issue.to_s => IssuesFinder + }.freeze def initialize(stage:, params: {}) @stage = stage - @params = params + @params = build_finder_params(params) end + # rubocop: disable CodeReuse/ActiveRecord def build - query = subject_class - query = filter_by_parent_model(query) - query = filter_by_time_range(query) + query = finder.execute query = stage.start_event.apply_query_customization(query) query = stage.end_event.apply_query_customization(query) query.where(duration_condition) end + # rubocop: enable CodeReuse/ActiveRecord private @@ -32,38 +35,33 @@ module Gitlab stage.end_event.timestamp_projection.gteq(stage.start_event.timestamp_projection) end - def filter_by_parent_model(query) - if parent_class.eql?(Project) - if subject_class.eql?(Issue) - query.where(project_id: stage.parent_id) - elsif subject_class.eql?(MergeRequest) - query.where(target_project_id: stage.parent_id) - else - raise ArgumentError, "unknown subject_class: #{subject_class}" - end - else - raise ArgumentError, "unknown parent_class: #{parent_class}" - end + def finder + FINDER_CLASSES.fetch(subject_class.to_s).new(params[:current_user], params) end - def filter_by_time_range(query) - from = params.fetch(:from, 30.days.ago) - to = params[:to] - - query = query.where(subject_table[:created_at].gteq(from)) - query = query.where(subject_table[:created_at].lteq(to)) if to - query + def parent_class + stage.parent.class end - def subject_table - subject_class.arel_table + def build_finder_params(params) + {}.tap do |finder_params| + finder_params[:current_user] = params[:current_user] + + add_parent_model_params!(finder_params) + add_time_range_params!(finder_params, params[:from], params[:to]) + end end - def parent_class - stage.parent.class + def add_parent_model_params!(finder_params) + raise(ArgumentError, "unknown parent_class: #{parent_class}") unless parent_class.eql?(Project) + + finder_params[:project_id] = stage.parent_id end - # rubocop: enable CodeReuse/ActiveRecord + def add_time_range_params!(finder_params, from, to) + finder_params[:created_after] = from || 30.days.ago + finder_params[:created_before] = to if to + end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb index 9fcaeadf351..6c0450ac9e5 100644 --- a/lib/gitlab/analytics/cycle_analytics/median.rb +++ b/lib/gitlab/analytics/cycle_analytics/median.rb @@ -11,12 +11,14 @@ module Gitlab @query = query end + # rubocop: disable CodeReuse/ActiveRecord def seconds - @query = @query.select(median_duration_in_seconds.as('median')) + @query = @query.select(median_duration_in_seconds.as('median')).reorder(nil) result = execute_query(@query).first || {} result['median'] || nil end + # rubocop: enable CodeReuse/ActiveRecord def days seconds ? seconds.fdiv(1.day) : nil diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index e8e269a88f0..e7352a23b99 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -12,13 +12,11 @@ module Gitlab MAPPINGS = { Issue => { - finder_class: IssuesFinder, serializer_class: AnalyticsIssueSerializer, includes_for_query: { project: [:namespace], author: [] }, columns_for_select: %I[title iid id created_at author_id project_id] }, MergeRequest => { - finder_class: MergeRequestsFinder, serializer_class: AnalyticsMergeRequestSerializer, includes_for_query: { target_project: [:namespace], author: [] }, columns_for_select: %I[title iid id created_at author_id state_id target_project_id] @@ -56,27 +54,12 @@ module Gitlab attr_reader :stage, :query, :params - def finder_query - MAPPINGS - .fetch(subject_class) - .fetch(:finder_class) - .new(params.fetch(:current_user), finder_params.fetch(stage.parent.class)) - .execute - end - def columns MAPPINGS.fetch(subject_class).fetch(:columns_for_select).map do |column_name| subject_class.arel_table[column_name] end end - # EE will override this to include Group rules - def finder_params - { - Project => { project_id: stage.parent_id } - } - end - def default_test_stage? stage.matches_with_stage_params?(Gitlab::Analytics::CycleAnalytics::DefaultStages.params_for_test_stage) end @@ -113,8 +96,7 @@ module Gitlab end def records - results = finder_query - .merge(ordered_and_limited_query) + results = ordered_and_limited_query .select(*columns, round_duration_to_seconds.as('total_time')) # using preloader instead of includes to avoid AR generating a large column list diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 2defbd26b98..3277ddd9f49 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -20,15 +20,16 @@ module Gitlab def rate_limits { issues_create: { threshold: -> { Gitlab::CurrentSettings.current_application_settings.issues_create_limit }, interval: 1.minute }, - project_export: { threshold: 1, interval: 5.minutes }, + project_export: { threshold: 30, interval: 5.minutes }, project_download_export: { threshold: 10, interval: 10.minutes }, project_repositories_archive: { threshold: 5, interval: 1.minute }, - project_generate_new_export: { threshold: 1, interval: 5.minutes }, + project_generate_new_export: { threshold: 30, interval: 5.minutes }, project_import: { threshold: 30, interval: 5.minutes }, play_pipeline_schedule: { threshold: 1, interval: 1.minute }, show_raw_controller: { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute }, - group_export: { threshold: 1, interval: 5.minutes }, - group_download_export: { threshold: 10, interval: 10.minutes } + group_export: { threshold: 30, interval: 5.minutes }, + group_download_export: { threshold: 10, interval: 10.minutes }, + group_import: { threshold: 30, interval: 5.minutes } }.freeze end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index b7e78189d37..93342fbad51 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -56,6 +56,7 @@ module Gitlab def find_user_from_job_token return unless route_authentication_setting[:job_token_allowed] + return find_user_from_basic_auth_job if route_authentication_setting[:job_token_allowed] == :basic_auth token = current_request.params[JOB_TOKEN_PARAM].presence || current_request.params[RUNNER_JOB_TOKEN_PARAM].presence || diff --git a/lib/gitlab/auth/ldap/person.rb b/lib/gitlab/auth/ldap/person.rb index e4a4900c37a..b3321c0b1fb 100644 --- a/lib/gitlab/auth/ldap/person.rb +++ b/lib/gitlab/auth/ldap/person.rb @@ -39,7 +39,7 @@ module Gitlab *config.attributes['name'], *config.attributes['email'], *config.attributes['username'] - ].compact.uniq + ].compact.uniq.reject(&:blank?) end def self.normalize_dn(dn) diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index 6d699d37a8c..1ca59aa827b 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -41,10 +41,6 @@ module Gitlab name.to_s.start_with?('ldap') end - def self.ultraauth_provider?(name) - name.to_s.eql?('ultraauth') - end - def self.sync_profile_from_provider?(provider) return true if ldap_provider?(provider) diff --git a/lib/gitlab/background_migration/.rubocop.yml b/lib/gitlab/background_migration/.rubocop.yml index 8242821cedc..50112a51675 100644 --- a/lib/gitlab/background_migration/.rubocop.yml +++ b/lib/gitlab/background_migration/.rubocop.yml @@ -15,7 +15,7 @@ Metrics/AbcSize: Metrics/PerceivedComplexity: Enabled: true -Metrics/LineLength: +Layout/LineLength: Enabled: true Details: > Long lines are very hard to read and make it more difficult to review diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index 263546bd132..bc113a1e33d 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -189,7 +189,7 @@ module Gitlab end def perform(start_id, stop_id) - Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) + Gitlab::Database.bulk_insert(:project_repositories, project_repositories(start_id, stop_id)) # rubocop:disable Gitlab/BulkInsert end private diff --git a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb index c652a5bb3fc..e750b8ca374 100644 --- a/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb +++ b/lib/gitlab/background_migration/fix_projects_without_prometheus_service.rb @@ -62,7 +62,7 @@ module Gitlab class PrometheusService < ActiveRecord::Base self.inheritance_column = :_type_disabled self.table_name = 'services' - default_scope { where(type: type) } + default_scope { where(type: type) } # rubocop:disable Cop/DefaultScope def self.type 'PrometheusService' diff --git a/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb b/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb new file mode 100644 index 00000000000..46921a070c3 --- /dev/null +++ b/lib/gitlab/background_migration/fix_ruby_object_in_audit_events.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Remove serialized Ruby object in audit_events + class FixRubyObjectInAuditEvents + def perform(start_id, stop_id) + end + end + end +end + +Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents.prepend_if_ee('EE::Gitlab::BackgroundMigration::FixRubyObjectInAuditEvents') diff --git a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb index 899f381e911..d2a9939b9ee 100644 --- a/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb +++ b/lib/gitlab/background_migration/migrate_fingerprint_sha256_within_keys.rb @@ -34,7 +34,7 @@ module Gitlab end end - Gitlab::Database.bulk_insert(TEMP_TABLE, fingerprints) + Gitlab::Database.bulk_insert(TEMP_TABLE, fingerprints) # rubocop:disable Gitlab/BulkInsert execute("ANALYZE #{TEMP_TABLE}") diff --git a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb index 956f9daa493..2bce5037d03 100644 --- a/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb +++ b/lib/gitlab/background_migration/migrate_issue_trackers_sensitive_data.rb @@ -65,7 +65,7 @@ module Gitlab next if service_ids.empty? migrated_ids += service_ids - Gitlab::Database.bulk_insert(table, data) + Gitlab::Database.bulk_insert(table, data) # rubocop:disable Gitlab/BulkInsert end return if migrated_ids.empty? diff --git a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb index 35bfc381180..fcbcaacb2d6 100644 --- a/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb +++ b/lib/gitlab/background_migration/populate_cluster_kubernetes_namespace_table.rb @@ -73,7 +73,7 @@ module Gitlab end def insert_into_cluster_kubernetes_namespace(rows) - Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name, + Gitlab::Database.bulk_insert(Migratable::KubernetesNamespace.table_name, # rubocop:disable Gitlab/BulkInsert rows, disable_quote: [:created_at, :updated_at]) end diff --git a/lib/gitlab/background_migration/populate_untracked_uploads.rb b/lib/gitlab/background_migration/populate_untracked_uploads.rb index d2924d10225..43698b7955f 100644 --- a/lib/gitlab/background_migration/populate_untracked_uploads.rb +++ b/lib/gitlab/background_migration/populate_untracked_uploads.rb @@ -95,7 +95,7 @@ module Gitlab file.to_h.merge(created_at: 'NOW()') end - Gitlab::Database.bulk_insert('uploads', + Gitlab::Database.bulk_insert('uploads', # rubocop:disable Gitlab/BulkInsert rows, disable_quote: :created_at) end diff --git a/lib/gitlab/background_migration/reset_merge_status.rb b/lib/gitlab/background_migration/reset_merge_status.rb index 447fec8903c..d040b4931be 100644 --- a/lib/gitlab/background_migration/reset_merge_status.rb +++ b/lib/gitlab/background_migration/reset_merge_status.rb @@ -7,7 +7,7 @@ module Gitlab class ResetMergeStatus def perform(from_id, to_id) relation = MergeRequest.where(id: from_id..to_id, - state: 'opened', + state_id: 1, # opened merge_status: 'can_be_merged') relation.update_all(merge_status: 'unchecked') diff --git a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb index cf0f582a2d4..d71a50a0af6 100644 --- a/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb +++ b/lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb @@ -25,7 +25,7 @@ module Gitlab mentions << mention_record unless mention_record.blank? end - Gitlab::Database.bulk_insert( + Gitlab::Database.bulk_insert( # rubocop:disable Gitlab/BulkInsert resource_user_mention_model.table_name, mentions, return_ids: true, diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/badge/coverage/report.rb index 15cccc6f287..0df6e858bf4 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/badge/coverage/report.rb @@ -7,12 +7,16 @@ module Gitlab # Test coverage report badge # class Report < Badge::Base - attr_reader :project, :ref, :job + attr_reader :project, :ref, :job, :customization - def initialize(project, ref, job = nil) + def initialize(project, ref, opts: { job: nil }) @project = project @ref = ref - @job = job + @job = opts[:job] + @customization = { + key_width: opts[:key_width].to_i, + key_text: opts[:key_text] + } @pipeline = @project.ci_pipelines.latest_successful_for_ref(@ref) end diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/badge/coverage/template.rb index 817dc28f84a..6b78825aefd 100644 --- a/lib/gitlab/badge/coverage/template.rb +++ b/lib/gitlab/badge/coverage/template.rb @@ -20,10 +20,16 @@ module Gitlab def initialize(badge) @entity = badge.entity @status = badge.status + @key_text = badge.customization.dig(:key_text) + @key_width = badge.customization.dig(:key_width) end def key_text - @entity.to_s + if @key_text && @key_text.size <= MAX_KEY_SIZE + @key_text + else + @entity.to_s + end end def value_text @@ -31,7 +37,11 @@ module Gitlab end def key_width - 62 + if @key_width && @key_width.between?(1, MAX_KEY_SIZE) + @key_width + else + 62 + end end def value_width diff --git a/lib/gitlab/badge/pipeline/status.rb b/lib/gitlab/badge/pipeline/status.rb index a403d839517..17f179f027d 100644 --- a/lib/gitlab/badge/pipeline/status.rb +++ b/lib/gitlab/badge/pipeline/status.rb @@ -7,11 +7,15 @@ module Gitlab # Pipeline status badge # class Status < Badge::Base - attr_reader :project, :ref + attr_reader :project, :ref, :customization - def initialize(project, ref) + def initialize(project, ref, opts: {}) @project = project @ref = ref + @customization = { + key_width: opts[:key_width].to_i, + key_text: opts[:key_text] + } @sha = @project.commit(@ref).try(:sha) end diff --git a/lib/gitlab/badge/pipeline/template.rb b/lib/gitlab/badge/pipeline/template.rb index 0d3d44135e7..781897fab4b 100644 --- a/lib/gitlab/badge/pipeline/template.rb +++ b/lib/gitlab/badge/pipeline/template.rb @@ -24,10 +24,16 @@ module Gitlab def initialize(badge) @entity = badge.entity @status = badge.status + @key_text = badge.customization.dig(:key_text) + @key_width = badge.customization.dig(:key_width) end def key_text - @entity.to_s + if @key_text && @key_text.size <= MAX_KEY_SIZE + @key_text + else + @entity.to_s + end end def value_text @@ -35,7 +41,11 @@ module Gitlab end def key_width - 62 + if @key_width && @key_width.between?(1, MAX_KEY_SIZE) + @key_width + else + 62 + end end def value_width diff --git a/lib/gitlab/badge/template.rb b/lib/gitlab/badge/template.rb index ed2ec50b197..97103e3f42c 100644 --- a/lib/gitlab/badge/template.rb +++ b/lib/gitlab/badge/template.rb @@ -6,6 +6,8 @@ module Gitlab # Abstract template class for badges # class Template + MAX_KEY_SIZE = 128 + def initialize(badge) @entity = badge.entity @status = badge.status diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index d8f9105d66d..5a9fad3be56 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -43,7 +43,7 @@ module Gitlab def store_pull_request_error(pull_request, ex) backtrace = Gitlab::BacktraceCleaner.clean_backtrace(ex.backtrace) - error = { type: :pull_request, iid: pull_request.iid, errors: ex.message, trace: backtrace, raw_response: pull_request.raw } + error = { type: :pull_request, iid: pull_request.iid, errors: ex.message, trace: backtrace, raw_response: pull_request.raw&.to_json } Gitlab::ErrorTracking.log_exception(ex, error) diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 7f2d2858149..ec94991157a 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -113,15 +113,18 @@ module Gitlab end end - # Sets multiple keys to a given value. + # Sets multiple keys to given values. # # mapping - A Hash mapping the cache keys to their values. + # key_prefix - prefix inserted before each key # timeout - The time after which the cache key should expire. - def self.write_multiple(mapping, timeout: TIMEOUT) + def self.write_multiple(mapping, key_prefix: nil, timeout: TIMEOUT) Redis::Cache.with do |redis| - redis.multi do |multi| + redis.pipelined do |multi| mapping.each do |raw_key, value| - multi.set(cache_key_for(raw_key), value, ex: timeout) + key = cache_key_for("#{key_prefix}#{raw_key}") + + multi.set(key, value, ex: timeout) end end end diff --git a/lib/gitlab/ci/build/releaser.rb b/lib/gitlab/ci/build/releaser.rb new file mode 100644 index 00000000000..ba6c7857e96 --- /dev/null +++ b/lib/gitlab/ci/build/releaser.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + class Releaser + BASE_COMMAND = 'release-cli create' + + attr_reader :config + + def initialize(config:) + @config = config + end + + def script + command = BASE_COMMAND.dup + config.each { |k, v| command.concat(" --#{k.to_s.dasherize} \"#{v}\"") } + + command + end + end + end + end +end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb index 48111ae5717..f8550b50905 100644 --- a/lib/gitlab/ci/build/step.rb +++ b/lib/gitlab/ci/build/step.rb @@ -20,6 +20,19 @@ module Gitlab end end + def from_release(job) + return unless Gitlab::Ci::Features.release_generation_enabled? + + release = job.options[:release] + return unless release + + self.new(:release).tap do |step| + step.script = Gitlab::Ci::Build::Releaser.new(config: job.options[:release]).script + step.timeout = job.metadata_timeout + step.when = WHEN_ON_SUCCESS + end + end + def from_after_script(job) after_script = job.options[:after_script] return unless after_script diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 1ea59491378..66050a7bbe0 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -28,7 +28,7 @@ module Gitlab in: %i[release], message: 'release features are not enabled' }, - unless: -> { Feature.enabled?(:ci_release_generation, default_enabled: false) } + unless: -> { Gitlab::Ci::Features.release_generation_enabled? } with_options allow_nil: true do validates :allow_failure, boolean: true diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 1a871e043a6..74736b24d73 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -12,9 +12,10 @@ module Gitlab include ::Gitlab::Config::Entry::Attributable ALLOWED_KEYS = - %i[junit codequality sast dependency_scanning container_scanning + %i[junit codequality sast secret_detection dependency_scanning container_scanning dast performance license_management license_scanning metrics lsif - dotenv cobertura terraform accessibility cluster_applications].freeze + dotenv cobertura terraform accessibility cluster_applications + requirements].freeze attributes ALLOWED_KEYS @@ -26,6 +27,7 @@ module Gitlab validates :junit, array_of_strings_or_string: true validates :codequality, array_of_strings_or_string: true validates :sast, array_of_strings_or_string: true + validates :secret_detection, array_of_strings_or_string: true validates :dependency_scanning, array_of_strings_or_string: true validates :container_scanning, array_of_strings_or_string: true validates :dast, array_of_strings_or_string: true @@ -39,6 +41,7 @@ module Gitlab validates :terraform, array_of_strings_or_string: true validates :accessibility, array_of_strings_or_string: true validates :cluster_applications, array_of_strings_or_string: true + validates :requirements, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 48f3d4fdd2f..a2eb31369c7 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -7,12 +7,40 @@ module Gitlab # module Features def self.artifacts_exclude_enabled? - ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: false) + ::Feature.enabled?(:ci_artifacts_exclude, default_enabled: true) end def self.ensure_scheduling_type_enabled? ::Feature.enabled?(:ci_ensure_scheduling_type, default_enabled: true) end + + def self.job_heartbeats_runner?(project) + ::Feature.enabled?(:ci_job_heartbeats_runner, project, default_enabled: true) + end + + def self.instance_level_variables_limit_enabled? + ::Feature.enabled?(:ci_instance_level_variables_limit, default_enabled: true) + end + + def self.pipeline_fixed_notifications? + ::Feature.enabled?(:ci_pipeline_fixed_notifications) + end + + def self.instance_variables_ui_enabled? + ::Feature.enabled?(:ci_instance_variables_ui, default_enabled: true) + end + + def self.composite_status?(project) + ::Feature.enabled?(:ci_composite_status, project, default_enabled: true) + end + + def self.atomic_processing?(project) + ::Feature.enabled?(:ci_atomic_processing, project, default_enabled: true) + end + + def self.release_generation_enabled? + ::Feature.enabled?(:ci_release_generation) + end end end end diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb index 26a18c6603e..19f724b79af 100644 --- a/lib/gitlab/ci/parsers/terraform/tfplan.rb +++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb @@ -8,15 +8,11 @@ module Gitlab TfplanParserError = Class.new(Gitlab::Ci::Parsers::ParserError) def parse!(json_data, terraform_reports, artifact:) - tfplan = Gitlab::Json.parse(json_data).tap do |parsed_data| - parsed_data['job_path'] = Gitlab::Routing.url_helpers.project_job_path( - artifact.job.project, artifact.job - ) - end + plan_data = Gitlab::Json.parse(json_data) - raise TfplanParserError, 'Tfplan missing required key' unless valid_supported_keys?(tfplan) + raise TfplanParserError, 'Tfplan missing required key' unless has_required_keys?(plan_data) - terraform_reports.add_plan(artifact.filename, tfplan) + terraform_reports.add_plan(artifact.job.id.to_s, tfplan(plan_data, artifact.job)) rescue JSON::ParserError raise TfplanParserError, 'JSON parsing failed' rescue @@ -25,8 +21,18 @@ module Gitlab private - def valid_supported_keys?(tfplan) - tfplan.keys == %w[create update delete job_path] + def has_required_keys?(plan_data) + (%w[create update delete] - plan_data.keys).empty? + end + + def tfplan(plan_data, artifact_job) + { + 'create' => plan_data['create'].to_i, + 'delete' => plan_data['delete'].to_i, + 'job_name' => artifact_job.options.dig(:artifacts, :name).to_s, + 'job_path' => Gitlab::Routing.url_helpers.project_job_path(artifact_job.project, artifact_job), + 'update' => plan_data['update'].to_i + } end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 73187401903..8118e7b2487 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -77,19 +77,18 @@ module Gitlab bridge&.parent_pipeline end - def duration_histogram - strong_memoize(:duration_histogram) do - name = :gitlab_ci_pipeline_creation_duration_seconds - comment = 'Pipeline creation duration' - labels = {} - buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] - - Gitlab::Metrics.histogram(name, comment, labels, buckets) - end + def metrics + @metrics ||= Chain::Metrics.new end def observe_creation_duration(duration) - duration_histogram.observe({}, duration.seconds) + metrics.pipeline_creation_duration_histogram + .observe({}, duration.seconds) + end + + def observe_pipeline_size(pipeline) + metrics.pipeline_size_histogram + .observe({ source: pipeline.source.to_s }, pipeline.total_size) end end end diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb new file mode 100644 index 00000000000..980ab2de9b0 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/metrics.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class Metrics + include Gitlab::Utils::StrongMemoize + + def pipeline_creation_duration_histogram + strong_memoize(:pipeline_creation_duration_histogram) do + name = :gitlab_ci_pipeline_creation_duration_seconds + comment = 'Pipeline creation duration' + labels = {} + buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 20.0, 50.0, 240.0] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + + def pipeline_size_histogram + strong_memoize(:pipeline_size_histogram) do + name = :gitlab_ci_pipeline_size_builds + comment = 'Pipeline size' + labels = { source: nil } + buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 2e177cfec7e..e48e79d561b 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -13,6 +13,7 @@ module Gitlab # Allocate next IID. This operation must be outside of transactions of pipeline creations. pipeline.ensure_project_iid! + pipeline.ensure_ci_ref! # Protect the pipeline. This is assigned in Populate instead of # Build to prevent erroring out on ambiguous refs. diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index a7c671e76d3..204c7725214 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -27,6 +27,7 @@ module Gitlab yield @pipeline, self if block_given? @command.observe_creation_duration(Time.now - @start) + @command.observe_pipeline_size(@pipeline) end end diff --git a/lib/gitlab/ci/reports/terraform_reports.rb b/lib/gitlab/ci/reports/terraform_reports.rb index f955d007daf..4b52c25d724 100644 --- a/lib/gitlab/ci/reports/terraform_reports.rb +++ b/lib/gitlab/ci/reports/terraform_reports.rb @@ -10,14 +10,6 @@ module Gitlab @plans = {} end - def pick(keys) - terraform_plans = plans.select do |key| - keys.include?(key) - end - - { plans: terraform_plans } - end - def add_plan(name, plan) plans[name] = plan end diff --git a/lib/gitlab/ci/status/bridge/failed.rb b/lib/gitlab/ci/status/bridge/failed.rb index de7446c238c..b0ab0992594 100644 --- a/lib/gitlab/ci/status/bridge/failed.rb +++ b/lib/gitlab/ci/status/bridge/failed.rb @@ -5,6 +5,14 @@ module Gitlab module Status module Bridge class Failed < Status::Build::Failed + private + + def failure_reason_message + [ + self.class.reasons.fetch(subject.failure_reason.to_sym), + subject.options[:downstream_errors] + ].flatten.compact.join(', ') + end end end end diff --git a/lib/gitlab/ci/status/core.rb b/lib/gitlab/ci/status/core.rb index ea773ee9944..4779c8d3d53 100644 --- a/lib/gitlab/ci/status/core.rb +++ b/lib/gitlab/ci/status/core.rb @@ -3,7 +3,7 @@ module Gitlab module Ci module Status - # Base abstract class fore core status + # Base abstract class for core status # class Core include Gitlab::Routing diff --git a/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml new file mode 100644 index 00000000000..82b2f5c035e --- /dev/null +++ b/lib/gitlab/ci/templates/AWS/Deploy-ECS.gitlab-ci.yml @@ -0,0 +1,13 @@ +stages: + - build + - test + - review + - deploy + - production + +variables: + AUTO_DEVOPS_PLATFORM_TARGET: ECS + +include: + - template: Jobs/Build.gitlab-ci.yml + - template: Jobs/Deploy/ECS.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 5017037fb5a..e37cd14d1d1 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -13,6 +13,7 @@ # * license_management: LICENSE_MANAGEMENT_DISABLED # * performance: PERFORMANCE_DISABLED # * sast: SAST_DISABLED +# * secret_detection: SECRET_DETECTION_DISABLED # * dependency_scanning: DEPENDENCY_SCANNING_DISABLED # * container_scanning: CONTAINER_SCANNING_DISABLED # * dast: DAST_DISABLED @@ -160,3 +161,4 @@ include: - template: Security/Dependency-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml - template: Security/License-Scanning.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml - template: Security/SAST.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml + - template: Security/Secret-Detection.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab-foss/blob/master/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml diff --git a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml index 82b2f5c035e..5f4bd631db6 100644 --- a/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Deploy-ECS.gitlab-ci.yml @@ -1,3 +1,18 @@ +# This template is deprecated and will be removed as part of GitLab 13.2! +# +# If you have referenced this template in your CI pipeline, please +# update your CI configuration by replacing the following occurrence(s): +# +# template: Deploy-ECS.gitlab-ci.yml +# +# with +# +# template: AWS/Deploy-ECS.gitlab-ci.yml +# +# -------------------- +# +# Documentation: https://docs.gitlab.com/ee/ci/cloud_deployment/#deploy-your-application-to-the-aws-elastic-container-service-ecs + stages: - build - test @@ -5,6 +20,9 @@ stages: - deploy - production +before_script: + - printf '\nWARNING!\nThis job includes "Deploy-ECS.gitlab-ci.yml". Please rename this to "AWS/Deploy-ECS.gitlab-ci.yml".\n' + variables: AUTO_DEVOPS_PLATFORM_TARGET: ECS diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index adbf9731e43..9a34f8cb113 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -1,11 +1,11 @@ performance: stage: performance - image: docker:19.03.8 + image: docker:19.03.11 allow_failure: true variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.8-dind + - docker:19.03.11-dind script: - | if ! docker info &>/dev/null; then diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 787f07521e0..b5550461482 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,10 +1,10 @@ build: stage: build - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.2.2" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v0.2.3" variables: DOCKER_TLS_CERTDIR: "" services: - - docker:19.03.8-dind + - docker:19.03.11-dind script: - | if [[ -z "$CI_COMMIT_TAG" ]]; then diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 24e75c56a75..bde6f185d3a 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -1,9 +1,9 @@ code_quality: stage: test - image: docker:19.03.8 + image: docker:19.03.11 allow_failure: true services: - - docker:19.03.8-dind + - docker:19.03.11-dind variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 5174aed04ba..bab4fae67f0 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.15.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.0" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 3fbae496896..97b5f3fd7f5 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.15.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v0.17.0" include: - template: Jobs/Deploy/ECS.gitlab-ci.yml @@ -177,6 +177,7 @@ production_manual: .manual_rollout_template: &manual_rollout_template <<: *rollout_template stage: production + resource_group: production allow_failure: true rules: - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""' diff --git a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml index 642f0ebeaf7..bb3d5526f3a 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy/ECS.gitlab-ci.yml @@ -1,3 +1,13 @@ +# WARNING (post-GitLab 13.0): +# +# This CI template should NOT be included in your own CI configuration files: +# 'review_ecs' and 'production_ecs' are two temporary names given to the jobs below. +# +# Should this template be included in your CI configuration, the upcoming name changes could +# then result in potentially breaking your future pipelines. +# +# More about including CI templates: https://docs.gitlab.com/ee/ci/yaml/#includetemplate + .deploy_to_ecs: image: 'registry.gitlab.com/gitlab-org/cloud-deploy/aws-ecs:latest' script: @@ -15,7 +25,9 @@ review_ecs: when: never - if: '$REVIEW_DISABLED' when: never - - if: '$CI_COMMIT_BRANCH != "master"' + - if: '$CI_COMMIT_BRANCH == "master"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' production_ecs: extends: .deploy_to_ecs @@ -27,4 +39,6 @@ production_ecs: when: never - if: '$CI_KUBERNETES_ACTIVE' when: never - - if: '$CI_COMMIT_BRANCH == "master"' + - if: '$CI_COMMIT_BRANCH != "master"' + when: never + - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH' diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 54a29b04d39..316647b5921 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.15.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.20.0" environment: name: production variables: @@ -19,12 +19,17 @@ apply: CROSSPLANE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/crossplane/values.yaml FLUENTD_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/fluentd/values.yaml KNATIVE_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/knative/values.yaml + POSTHOG_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/posthog/values.yaml + FALCO_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/falco/values.yaml + APPARMOR_VALUES_FILE: $CI_PROJECT_DIR/.gitlab/managed-apps/apparmor/values.yaml script: - gitlab-managed-apps /usr/local/share/gitlab-managed-apps/helmfile.yaml only: refs: - master artifacts: + reports: + cluster_applications: gl-cluster-applications.json when: on_failure paths: - tiller.log diff --git a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml index a25dc38e4e7..f35470367cc 100644 --- a/lib/gitlab/ci/templates/Rust.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Rust.gitlab-ci.yml @@ -20,4 +20,4 @@ image: "rust:latest" test:cargo: script: - rustc --version && cargo --version # Print version info for debugging - - cargo test --all --verbose + - cargo test --workspace --verbose diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 616966b4f04..fa8ccb7cf93 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -13,6 +13,7 @@ variables: DS_ANALYZER_IMAGE_PREFIX: "$SECURE_ANALYZERS_PREFIX" DS_DEFAULT_ANALYZERS: "bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python" + DS_EXCLUDED_PATHS: "spec, test, tests, tmp" DS_MAJOR_VERSION: 2 DS_DISABLE_DIND: "true" @@ -125,6 +126,7 @@ gemnasium-maven-dependency_scanning: $DS_DEFAULT_ANALYZERS =~ /gemnasium-maven/ exists: - '{build.gradle,*/build.gradle,*/*/build.gradle}' + - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' - '{build.sbt,*/build.sbt,*/*/build.sbt}' - '{pom.xml,*/pom.xml,*/*/pom.xml}' diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index b86014c1ebc..b0c75b0aab0 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -19,6 +19,7 @@ license_scanning: entrypoint: [""] variables: LM_REPORT_FILE: gl-license-scanning-report.json + LM_REPORT_VERSION: '2.1' SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD allow_failure: true script: diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 47f68118ee0..ec7b34d17b5 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -13,6 +13,7 @@ variables: SAST_ANALYZER_IMAGE_PREFIX: "$SECURE_ANALYZERS_PREFIX" SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec" + SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" SAST_ANALYZER_IMAGE_TAG: 2 SAST_DISABLE_DIND: "true" SCAN_KUBERNETES_MANIFESTS: "false" @@ -80,10 +81,9 @@ brakeman-sast: - if: $SAST_DISABLED || $SAST_DISABLE_DIND == 'false' when: never - if: $CI_COMMIT_BRANCH && - $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /brakeman/ exists: - - '**/*.rb' + - 'config/routes.rb' eslint-sast: extends: .sast-analyzer @@ -149,7 +149,7 @@ nodejs-scan-sast: $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ exists: - - '**/*.js' + - 'package.json' phpcs-security-audit-sast: extends: .sast-analyzer @@ -213,8 +213,7 @@ sobelow-sast: $GITLAB_FEATURES =~ /\bsast\b/ && $SAST_DEFAULT_ANALYZERS =~ /sobelow/ exists: - - '**/*.ex' - - '**/*.exs' + - 'mix.exs' spotbugs-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml new file mode 100644 index 00000000000..e18f89cadd7 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -0,0 +1,24 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/secret_detection +# +# Configure the scanning tool through the environment variables. +# List of the variables: https://gitlab.com/gitlab-org/security-products/secret_detection#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + SECRETS_ANALYZER_VERSION: "3" + +secret_detection: + stage: test + image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION" + services: [] + rules: + - if: $SECRET_DETECTION_DISABLED + when: never + - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bsecret_detection\b/ + when: on_success + artifacts: + reports: + secret_detection: gl-secret-detection-report.json + script: + - /analyzer run diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index a0832718214..377c72e8031 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -40,7 +40,6 @@ plan: - terraform plan -out=$PLAN - "terraform show --json $PLAN | convert_report > $JSON_PLAN_FILE" artifacts: - name: plan paths: - $PLAN reports: diff --git a/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml new file mode 100644 index 00000000000..77a1b57d92f --- /dev/null +++ b/lib/gitlab/ci/templates/Verify/FailFast.gitlab-ci.yml @@ -0,0 +1,17 @@ +rspec-rails-modified-path-specs: + stage: .pre + rules: + - if: $CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "merge_train" + changes: ["**/*.rb"] + script: + - gem install test_file_finder + - spec_files=$(tff $(git diff --name-only "$CI_MERGE_REQUEST_TARGET_BRANCH_SHA..$CI_MERGE_REQUEST_SOURCE_BRANCH_SHA")) + - | + if [ -n "$spec_files" ] + then + bundle install + bundle exec rspec -- $spec_files + else + echo "No relevant spec files found by tff" + exit 0 + fi diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 4e83826b249..f76aacc2d19 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -147,7 +147,7 @@ module Gitlab raise AlreadyArchivedError, 'Could not write to the archived trace' elsif current_path File.open(current_path, mode) - elsif Feature.enabled?('ci_enable_live_trace', job.project) + elsif Feature.enabled?(:ci_enable_live_trace, job.project) Gitlab::Ci::Trace::ChunkedIO.new(job) else File.open(ensure_path, mode) diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 5816ac3bc54..6a9b7b2fc85 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -88,7 +88,7 @@ module Gitlab end def release(job) - job[:release] if Feature.enabled?(:ci_release_generation, default_enabled: false) + job[:release] if Gitlab::Ci::Features.release_generation_enabled? end def stage_builds_attributes(stage) diff --git a/lib/gitlab/code_navigation_path.rb b/lib/gitlab/code_navigation_path.rb index 57aeb6c4fb2..faf623faccf 100644 --- a/lib/gitlab/code_navigation_path.rb +++ b/lib/gitlab/code_navigation_path.rb @@ -13,7 +13,7 @@ module Gitlab end def full_json_path_for(path) - return if Feature.disabled?(:code_navigation, project) + return unless Feature.enabled?(:code_navigation, project, default_enabled: true) return unless build raw_project_job_artifacts_path(project, build, path: "lsif/#{path}.json", file_type: :lsif) diff --git a/lib/gitlab/config/loader/yaml.rb b/lib/gitlab/config/loader/yaml.rb index 4cedab1545c..e001742a7f8 100644 --- a/lib/gitlab/config/loader/yaml.rb +++ b/lib/gitlab/config/loader/yaml.rb @@ -21,11 +21,15 @@ module Gitlab hash? && !too_big? end - def load! + def load_raw! raise DataTooLargeError, 'The parsed YAML is too big' if too_big? raise Loader::FormatError, 'Invalid configuration format' unless hash? - @config.deep_symbolize_keys + @config + end + + def load! + @symbolized_config ||= load_raw!.deep_symbolize_keys end private diff --git a/lib/gitlab/contributions_calendar.rb b/lib/gitlab/contributions_calendar.rb index 5b0b91de5da..4e430d8937d 100644 --- a/lib/gitlab/contributions_calendar.rb +++ b/lib/gitlab/contributions_calendar.rb @@ -24,13 +24,13 @@ module Gitlab # project_features for the (currently) 3 different contribution types date_from = 1.year.ago repo_events = event_counts(date_from, :repository) - .having(action: Event::PUSHED) + .having(action: :pushed) issue_events = event_counts(date_from, :issues) - .having(action: [Event::CREATED, Event::CLOSED], target_type: "Issue") + .having(action: [:created, :closed], target_type: "Issue") mr_events = event_counts(date_from, :merge_requests) - .having(action: [Event::MERGED, Event::CREATED, Event::CLOSED], target_type: "MergeRequest") + .having(action: [:merged, :created, :closed], target_type: "MergeRequest") note_events = event_counts(date_from, :merge_requests) - .having(action: [Event::COMMENTED]) + .having(action: :commented) events = Event .from_union([repo_events, issue_events, mr_events, note_events]) diff --git a/lib/gitlab/cycle_analytics/summary/commit.rb b/lib/gitlab/cycle_analytics/summary/commit.rb index 1f426b81800..1dc9d5de966 100644 --- a/lib/gitlab/cycle_analytics/summary/commit.rb +++ b/lib/gitlab/cycle_analytics/summary/commit.rb @@ -5,7 +5,7 @@ module Gitlab module Summary class Commit < Base def title - n_('Commit', 'Commits', value) + n_('Commit', 'Commits', value.to_i) end def value diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index 8544ea1a91e..5125c8e64ee 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -5,7 +5,7 @@ module Gitlab module Summary class Deploy < Base def title - n_('Deploy', 'Deploys', value) + n_('Deploy', 'Deploys', value.to_i) end def value diff --git a/lib/gitlab/cycle_analytics/summary/issue.rb b/lib/gitlab/cycle_analytics/summary/issue.rb index ce7788590b9..462fd4c2d3d 100644 --- a/lib/gitlab/cycle_analytics/summary/issue.rb +++ b/lib/gitlab/cycle_analytics/summary/issue.rb @@ -12,7 +12,7 @@ module Gitlab end def title - n_('New Issue', 'New Issues', value) + n_('New Issue', 'New Issues', value.to_i) end def value diff --git a/lib/gitlab/cycle_analytics/summary/value.rb b/lib/gitlab/cycle_analytics/summary/value.rb index ce32132e048..e443e037517 100644 --- a/lib/gitlab/cycle_analytics/summary/value.rb +++ b/lib/gitlab/cycle_analytics/summary/value.rb @@ -15,6 +15,14 @@ module Gitlab end class None < self + def raw_value + 0 + end + + def to_i + raw_value + end + def to_s '-' end @@ -28,6 +36,10 @@ module Gitlab def to_s value.zero? ? '0' : value.to_s end + + def to_i + raw_value + end end class PrettyNumeric < Numeric diff --git a/lib/gitlab/danger/emoji_checker.rb b/lib/gitlab/danger/emoji_checker.rb index a2867087428..e31a6ae5011 100644 --- a/lib/gitlab/danger/emoji_checker.rb +++ b/lib/gitlab/danger/emoji_checker.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative '../json' +require 'json' module Gitlab module Danger @@ -25,8 +25,8 @@ module Gitlab )}x.freeze def initialize - names = Gitlab::Json.parse(File.read(DIGESTS)).keys + - Gitlab::Json.parse(File.read(ALIASES)).keys + names = JSON.parse(File.read(DIGESTS)).keys + + JSON.parse(File.read(ALIASES)).keys @emoji = names.map { |name| ":#{name}:" } end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index 0f0af5f777b..327418ad100 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -142,6 +142,7 @@ module Gitlab %r{Dangerfile\z} => :engineering_productivity, %r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity, %r{\A(ee/)?scripts/} => :engineering_productivity, + %r{\A\.codeclimate\.yml\z} => :engineering_productivity, %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, %r{\A(ee/)?(bin|config|generator_templates|lib|rubocop)/} => :backend, @@ -151,6 +152,7 @@ module Gitlab %r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend, %r{\A[A-Z_]+_VERSION\z} => :backend, %r{\A\.rubocop(_todo)?\.yml\z} => :backend, + %r{\Afile_hooks/} => :backend, %r{\A(ee/)?qa/} => :qa, diff --git a/lib/gitlab/danger/request_helper.rb b/lib/gitlab/danger/request_helper.rb index ef51c3f2052..06da4ed9ad3 100644 --- a/lib/gitlab/danger/request_helper.rb +++ b/lib/gitlab/danger/request_helper.rb @@ -16,7 +16,7 @@ module Gitlab raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}" end - Gitlab::Json.parse(rsp.body) + JSON.parse(rsp.body) end end end diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb index dbf42912882..9f7980dc20a 100644 --- a/lib/gitlab/danger/roulette.rb +++ b/lib/gitlab/danger/roulette.rb @@ -6,6 +6,46 @@ module Gitlab module Danger module Roulette ROULETTE_DATA_URL = 'https://about.gitlab.com/roulette.json' + OPTIONAL_CATEGORIES = [:qa, :test].freeze + + Spin = Struct.new(:category, :reviewer, :maintainer, :optional_role) + + # Assigns GitLab team members to be reviewer and maintainer + # for each change category that a Merge Request contains. + # + # @return [Array<Spin>] + def spin(project, categories, branch_name) + team = + begin + project_team(project) + rescue => err + warn("Reviewer roulette failed to load team data: #{err.message}") + [] + end + + canonical_branch_name = canonical_branch_name(branch_name) + + spin_per_category = categories.each_with_object({}) do |category, memo| + memo[category] = spin_for_category(team, project, category, canonical_branch_name) + end + + spin_per_category.map do |category, spin| + case category + when :test + if spin.reviewer.nil? + # Fetch an already picked backend reviewer, or pick one otherwise + spin.reviewer = spin_per_category[:backend]&.reviewer || spin_for_category(team, project, :backend, canonical_branch_name).reviewer + end + when :engineering_productivity + if spin.maintainer.nil? + # Fetch an already picked backend maintainer, or pick one otherwise + spin.maintainer = spin_per_category[:backend]&.maintainer || spin_for_category(team, project, :backend, canonical_branch_name).maintainer + end + end + + spin + end + end # Looks up the current list of GitLab team members and parses it into a # useful form @@ -58,6 +98,33 @@ module Gitlab def mr_author?(person) person.username == gitlab.mr_author end + + def spin_role_for_category(team, role, project, category) + team.select do |member| + member.public_send("#{role}?", project, category, gitlab.mr_labels) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def spin_for_category(team, project, category, branch_name) + reviewers, traintainers, maintainers = + %i[reviewer traintainer maintainer].map do |role| + spin_role_for_category(team, role, project, category) + end + + # TODO: take CODEOWNERS into account? + # https://gitlab.com/gitlab-org/gitlab/issues/26723 + + # Make traintainers have triple the chance to be picked as a reviewer + random = new_random(branch_name) + reviewer = spin_for_person(reviewers + traintainers + traintainers, random: random) + maintainer = spin_for_person(maintainers, random: random) + + Spin.new(category, reviewer, maintainer).tap do |spin| + if OPTIONAL_CATEGORIES.include?(category) + spin.optional_role = :maintainer + end + end + end end end end diff --git a/lib/gitlab/data_builder/alert.rb b/lib/gitlab/data_builder/alert.rb new file mode 100644 index 00000000000..e34bdeea799 --- /dev/null +++ b/lib/gitlab/data_builder/alert.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module DataBuilder + module Alert + extend self + + def build(alert) + { + object_kind: 'alert', + object_attributes: hook_attrs(alert) + } + end + + def hook_attrs(alert) + { + title: alert.title, + url: Gitlab::Routing.url_helpers.details_project_alert_management_url(alert.project, alert.iid), + severity: alert.severity, + events: alert.events, + status: alert.status_name, + started_at: alert.started_at + } + end + end + end +end diff --git a/lib/gitlab/data_builder/note.rb b/lib/gitlab/data_builder/note.rb index 2c4ef73a688..73518d36d43 100644 --- a/lib/gitlab/data_builder/note.rb +++ b/lib/gitlab/data_builder/note.rb @@ -55,7 +55,7 @@ module Gitlab end def build_base_data(project, user, note) - event_type = note.confidential? ? 'confidential_note' : 'note' + event_type = note.confidential?(include_noteable: true) ? 'confidential_note' : 'note' base_data = { object_kind: "note", diff --git a/lib/gitlab/database/custom_structure.rb b/lib/gitlab/database/custom_structure.rb new file mode 100644 index 00000000000..c5a76c5a787 --- /dev/null +++ b/lib/gitlab/database/custom_structure.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class CustomStructure + CUSTOM_DUMP_FILE = 'db/gitlab_structure.sql' + + def dump + File.open(self.class.custom_dump_filepath, 'wb') do |io| + io << "-- this file tracks custom GitLab data, such as foreign keys referencing partitioned tables\n" + io << "-- more details can be found in the issue: https://gitlab.com/gitlab-org/gitlab/-/issues/201872\n" + io << "SET search_path=public;\n\n" + + dump_partitioned_foreign_keys(io) if partitioned_foreign_keys_exist? + end + end + + def self.custom_dump_filepath + Rails.root.join(CUSTOM_DUMP_FILE) + end + + private + + def dump_partitioned_foreign_keys(io) + io << "COPY partitioned_foreign_keys (#{partitioned_fk_columns.join(", ")}) FROM STDIN;\n" + + PartitioningMigrationHelpers::PartitionedForeignKey.find_each do |fk| + io << fk.attributes.values_at(*partitioned_fk_columns).join("\t") << "\n" + end + io << "\\.\n" + end + + def partitioned_foreign_keys_exist? + return false unless PartitioningMigrationHelpers::PartitionedForeignKey.table_exists? + + PartitioningMigrationHelpers::PartitionedForeignKey.exists? + end + + def partitioned_fk_columns + @partitioned_fk_columns ||= PartitioningMigrationHelpers::PartitionedForeignKey.column_names + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 96be057f77e..fd09c31e994 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -3,6 +3,8 @@ module Gitlab module Database module MigrationHelpers + # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS + MAX_IDENTIFIER_NAME_LENGTH = 63 BACKGROUND_MIGRATION_BATCH_SIZE = 1000 # Number of rows to process per job BACKGROUND_MIGRATION_JOB_BUFFER_SIZE = 1000 # Number of jobs to bulk queue at a time @@ -1209,6 +1211,8 @@ into similar problems in the future (e.g. when new tables are created). # # rubocop:disable Gitlab/RailsLogger def add_check_constraint(table, check, constraint_name, validate: true) + validate_check_constraint_name!(constraint_name) + # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. if transaction_open? @@ -1244,6 +1248,8 @@ into similar problems in the future (e.g. when new tables are created). end def validate_check_constraint(table, constraint_name) + validate_check_constraint_name!(constraint_name) + unless check_constraint_exists?(table, constraint_name) raise missing_schema_object_message(table, "check constraint", constraint_name) end @@ -1256,6 +1262,8 @@ into similar problems in the future (e.g. when new tables are created). end def remove_check_constraint(table, constraint_name) + validate_check_constraint_name!(constraint_name) + # DROP CONSTRAINT requires an EXCLUSIVE lock # Use with_lock_retries to make sure that this will not timeout with_lock_retries do @@ -1330,6 +1338,12 @@ into similar problems in the future (e.g. when new tables are created). private + def validate_check_constraint_name!(constraint_name) + if constraint_name.to_s.length > MAX_IDENTIFIER_NAME_LENGTH + raise "The maximum allowed constraint name is #{MAX_IDENTIFIER_NAME_LENGTH} characters" + end + end + def statement_timeout_disabled? # This is a string of the form "100ms" or "0" when disabled connection.select_value('SHOW statement_timeout') == "0" diff --git a/lib/gitlab/database/partitioning_migration_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers.rb index 55649ebbf8a..881177a195e 100644 --- a/lib/gitlab/database/partitioning_migration_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers.rb @@ -3,120 +3,8 @@ module Gitlab module Database module PartitioningMigrationHelpers - include SchemaHelpers - - def add_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id, on_delete: :cascade) - cascade_delete = extract_cascade_option(on_delete) - - update_foreign_keys(from_table, to_table, column, primary_key, cascade_delete) do |current_keys, existing_key, specified_key| - if existing_key.nil? - unless specified_key.save - raise "failed to create foreign key: #{specified_key.errors.full_messages.to_sentence}" - end - - current_keys << specified_key - else - Rails.logger.warn "foreign key not added because it already exists: #{specified_key}" # rubocop:disable Gitlab/RailsLogger - current_keys - end - end - end - - def remove_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id) - update_foreign_keys(from_table, to_table, column, primary_key) do |current_keys, existing_key, specified_key| - if existing_key - existing_key.destroy! - current_keys.delete(existing_key) - else - Rails.logger.warn "foreign key not removed because it doesn't exist: #{specified_key}" # rubocop:disable Gitlab/RailsLogger - end - - current_keys - end - end - - def fk_function_name(table) - object_name(table, 'fk_cascade_function') - end - - def fk_trigger_name(table) - object_name(table, 'fk_cascade_trigger') - end - - private - - def fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete) - PartitionedForeignKey.new(from_table: from_table.to_s, to_table: to_table.to_s, from_column: from_column.to_s, - to_column: to_column.to_s, cascade_delete: cascade_delete) - end - - def update_foreign_keys(from_table, to_table, from_column, to_column, cascade_delete = nil) - if transaction_open? - raise 'partitioned foreign key operations can not be run inside a transaction block, ' \ - 'you can disable transaction blocks by calling disable_ddl_transaction! ' \ - 'in the body of your migration class' - end - - from_column ||= "#{to_table.to_s.singularize}_id" - specified_key = fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete) - - current_keys = PartitionedForeignKey.by_referenced_table(to_table).to_a - existing_key = find_existing_key(current_keys, specified_key) - - final_keys = yield current_keys, existing_key, specified_key - - fn_name = fk_function_name(to_table) - trigger_name = fk_trigger_name(to_table) - - with_lock_retries do - drop_trigger(to_table, trigger_name, if_exists: true) - - if final_keys.empty? - drop_function(fn_name, if_exists: true) - else - create_or_replace_fk_function(fn_name, final_keys) - create_function_trigger(trigger_name, fn_name, fires: "AFTER DELETE ON #{to_table}") - end - end - end - - def extract_cascade_option(on_delete) - case on_delete - when :cascade then true - when :nullify then false - else raise ArgumentError, "invalid option #{on_delete} for :on_delete" - end - end - - def with_lock_retries(&block) - Gitlab::Database::WithLockRetries.new({ - klass: self.class, - logger: Gitlab::BackgroundMigration::Logger - }).run(&block) - end - - def find_existing_key(keys, key) - keys.find { |k| k.from_table == key.from_table && k.from_column == key.from_column } - end - - def create_or_replace_fk_function(fn_name, fk_specs) - create_trigger_function(fn_name, replace: true) do - cascade_statements = build_cascade_statements(fk_specs) - cascade_statements << 'RETURN OLD;' - - cascade_statements.join("\n") - end - end - - def build_cascade_statements(foreign_keys) - foreign_keys.map do |fks| - if fks.cascade_delete? - "DELETE FROM #{fks.from_table} WHERE #{fks.from_column} = OLD.#{fks.to_column};" - else - "UPDATE #{fks.from_table} SET #{fks.from_column} = NULL WHERE #{fks.from_column} = OLD.#{fks.to_column};" - end - end - end + include ForeignKeyHelpers + include TableManagementHelpers end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb new file mode 100644 index 00000000000..9e687009cd7 --- /dev/null +++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PartitioningMigrationHelpers + module ForeignKeyHelpers + include ::Gitlab::Database::SchemaHelpers + + # Creates a "foreign key" that references a partitioned table. Because foreign keys referencing partitioned + # tables are not supported in PG11, this does not create a true database foreign key, but instead implements the + # same functionality at the database level by using triggers. + # + # Example: + # + # add_partitioned_foreign_key :issues, :projects + # + # Available options: + # + # :column - name of the referencing column (otherwise inferred from the referenced table name) + # :primary_key - name of the primary key in the referenced table (defaults to id) + # :on_delete - supports either :cascade for ON DELETE CASCADE or :nullify for ON DELETE SET NULL + # + def add_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id, on_delete: :cascade) + cascade_delete = extract_cascade_option(on_delete) + + update_foreign_keys(from_table, to_table, column, primary_key, cascade_delete) do |current_keys, existing_key, specified_key| + if existing_key.nil? + unless specified_key.save + raise "failed to create foreign key: #{specified_key.errors.full_messages.to_sentence}" + end + + current_keys << specified_key + else + Rails.logger.warn "foreign key not added because it already exists: #{specified_key}" # rubocop:disable Gitlab/RailsLogger + current_keys + end + end + end + + # Drops a "foreign key" that references a partitioned table. This method ONLY applies to foreign keys previously + # created through the `add_partitioned_foreign_key` method. Standard database foreign keys should be managed + # through the familiar Rails helpers. + # + # Example: + # + # remove_partitioned_foreign_key :issues, :projects + # + # Available options: + # + # :column - name of the referencing column (otherwise inferred from the referenced table name) + # :primary_key - name of the primary key in the referenced table (defaults to id) + # + def remove_partitioned_foreign_key(from_table, to_table, column: nil, primary_key: :id) + update_foreign_keys(from_table, to_table, column, primary_key) do |current_keys, existing_key, specified_key| + if existing_key + existing_key.delete + current_keys.delete(existing_key) + else + Rails.logger.warn "foreign key not removed because it doesn't exist: #{specified_key}" # rubocop:disable Gitlab/RailsLogger + end + + current_keys + end + end + + private + + def fk_function_name(table) + object_name(table, 'fk_cascade_function') + end + + def fk_trigger_name(table) + object_name(table, 'fk_cascade_trigger') + end + + def fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete) + PartitionedForeignKey.new(from_table: from_table.to_s, to_table: to_table.to_s, from_column: from_column.to_s, + to_column: to_column.to_s, cascade_delete: cascade_delete) + end + + def update_foreign_keys(from_table, to_table, from_column, to_column, cascade_delete = nil) + assert_not_in_transaction_block(scope: 'partitioned foreign key') + + from_column ||= "#{to_table.to_s.singularize}_id" + specified_key = fk_from_spec(from_table, to_table, from_column, to_column, cascade_delete) + + current_keys = PartitionedForeignKey.by_referenced_table(to_table).to_a + existing_key = find_existing_key(current_keys, specified_key) + + final_keys = yield current_keys, existing_key, specified_key + + fn_name = fk_function_name(to_table) + trigger_name = fk_trigger_name(to_table) + + with_lock_retries do + drop_trigger(to_table, trigger_name, if_exists: true) + + if final_keys.empty? + drop_function(fn_name, if_exists: true) + else + create_or_replace_fk_function(fn_name, final_keys) + create_trigger(trigger_name, fn_name, fires: "AFTER DELETE ON #{to_table}") + end + end + end + + def extract_cascade_option(on_delete) + case on_delete + when :cascade then true + when :nullify then false + else raise ArgumentError, "invalid option #{on_delete} for :on_delete" + end + end + + def find_existing_key(keys, key) + keys.find { |k| k.from_table == key.from_table && k.from_column == key.from_column } + end + + def create_or_replace_fk_function(fn_name, fk_specs) + create_trigger_function(fn_name, replace: true) do + cascade_statements = build_cascade_statements(fk_specs) + cascade_statements << 'RETURN OLD;' + + cascade_statements.join("\n") + end + end + + def build_cascade_statements(foreign_keys) + foreign_keys.map do |fks| + if fks.cascade_delete? + "DELETE FROM #{fks.from_table} WHERE #{fks.from_column} = OLD.#{fks.to_column};" + else + "UPDATE #{fks.from_table} SET #{fks.from_column} = NULL WHERE #{fks.from_column} = OLD.#{fks.to_column};" + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb new file mode 100644 index 00000000000..f77fbe98df1 --- /dev/null +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -0,0 +1,198 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PartitioningMigrationHelpers + module TableManagementHelpers + include ::Gitlab::Database::SchemaHelpers + + WHITELISTED_TABLES = %w[audit_events].freeze + ERROR_SCOPE = 'table partitioning' + + # Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column. + # One partition is created per month between the given `min_date` and `max_date`. + # + # A copy of the original table is required as PG currently does not support partitioning existing tables. + # + # Example: + # + # partition_table_by_date :audit_events, :created_at, min_date: Date.new(2020, 1), max_date: Date.new(2020, 6) + # + # Required options are: + # :min_date - a date specifying the lower bounds of the partition range + # :max_date - a date specifying the upper bounds of the partitioning range + # + def partition_table_by_date(table_name, column_name, min_date:, max_date:) + assert_table_is_whitelisted(table_name) + assert_not_in_transaction_block(scope: ERROR_SCOPE) + + raise "max_date #{max_date} must be greater than min_date #{min_date}" if min_date >= max_date + + primary_key = connection.primary_key(table_name) + raise "primary key not defined for #{table_name}" if primary_key.nil? + + partition_column = find_column_definition(table_name, column_name) + raise "partition column #{column_name} does not exist on #{table_name}" if partition_column.nil? + + new_table_name = partitioned_table_name(table_name) + create_range_partitioned_copy(new_table_name, table_name, partition_column, primary_key) + create_daterange_partitions(new_table_name, partition_column.name, min_date, max_date) + create_sync_trigger(table_name, new_table_name, primary_key) + end + + # Clean up a partitioned copy of an existing table. This deletes the partitioned table and all partitions. + # + # Example: + # + # drop_partitioned_table_for :audit_events + # + def drop_partitioned_table_for(table_name) + assert_table_is_whitelisted(table_name) + assert_not_in_transaction_block(scope: ERROR_SCOPE) + + with_lock_retries do + trigger_name = sync_trigger_name(table_name) + drop_trigger(table_name, trigger_name) + end + + function_name = sync_function_name(table_name) + drop_function(function_name) + + part_table_name = partitioned_table_name(table_name) + drop_table(part_table_name) + end + + private + + def assert_table_is_whitelisted(table_name) + return if WHITELISTED_TABLES.include?(table_name.to_s) + + raise "partitioning helpers are in active development, and #{table_name} is not whitelisted for use, " \ + "for more information please contact the database team" + end + + def partitioned_table_name(table) + tmp_table_name("#{table}_part") + end + + def sync_function_name(table) + object_name(table, 'table_sync_function') + end + + def sync_trigger_name(table) + object_name(table, 'table_sync_trigger') + end + + def find_column_definition(table, column) + connection.columns(table).find { |c| c.name == column.to_s } + end + + def create_range_partitioned_copy(table_name, template_table_name, partition_column, primary_key) + if table_exists?(table_name) + # rubocop:disable Gitlab/RailsLogger + Rails.logger.warn "Partitioned table not created because it already exists" \ + " (this may be due to an aborted migration or similar): table_name: #{table_name} " + # rubocop:enable Gitlab/RailsLogger + return + end + + tmp_column_name = object_name(partition_column.name, 'partition_key') + transaction do + execute(<<~SQL) + CREATE TABLE #{table_name} ( + LIKE #{template_table_name} INCLUDING ALL EXCLUDING INDEXES, + #{tmp_column_name} #{partition_column.sql_type} NOT NULL, + PRIMARY KEY (#{[primary_key, tmp_column_name].join(", ")}) + ) PARTITION BY RANGE (#{tmp_column_name}) + SQL + + remove_column(table_name, partition_column.name) + rename_column(table_name, tmp_column_name, partition_column.name) + change_column_default(table_name, primary_key, nil) + + if column_of_type?(table_name, primary_key, :integer) + # Default to int8 primary keys to prevent overflow + change_column(table_name, primary_key, :bigint) + end + end + end + + def column_of_type?(table_name, column, type) + find_column_definition(table_name, column).type == type + end + + def create_daterange_partitions(table_name, column_name, min_date, max_date) + min_date = min_date.beginning_of_month.to_date + max_date = max_date.next_month.beginning_of_month.to_date + + create_range_partition_safely("#{table_name}_000000", table_name, 'MINVALUE', to_sql_date_literal(min_date)) + + while min_date < max_date + partition_name = "#{table_name}_#{min_date.strftime('%Y%m')}" + next_date = min_date.next_month + lower_bound = to_sql_date_literal(min_date) + upper_bound = to_sql_date_literal(next_date) + + create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound) + min_date = next_date + end + end + + def to_sql_date_literal(date) + connection.quote(date.strftime('%Y-%m-%d')) + end + + def create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound) + if table_exists?(partition_name) + # rubocop:disable Gitlab/RailsLogger + Rails.logger.warn "Partition not created because it already exists" \ + " (this may be due to an aborted migration or similar): partition_name: #{partition_name}" + # rubocop:enable Gitlab/RailsLogger + return + end + + create_range_partition(partition_name, table_name, lower_bound, upper_bound) + end + + def create_sync_trigger(source_table, target_table, unique_key) + function_name = sync_function_name(source_table) + trigger_name = sync_trigger_name(source_table) + + with_lock_retries do + create_sync_function(function_name, target_table, unique_key) + create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table} table") + + create_trigger(trigger_name, function_name, fires: "AFTER INSERT OR UPDATE OR DELETE ON #{source_table}") + end + end + + def create_sync_function(name, target_table, unique_key) + delimiter = ",\n " + column_names = connection.columns(target_table).map(&:name) + set_statements = build_set_statements(column_names, unique_key) + insert_values = column_names.map { |name| "NEW.#{name}" } + + create_trigger_function(name, replace: false) do + <<~SQL + IF (TG_OP = 'DELETE') THEN + DELETE FROM #{target_table} where #{unique_key} = OLD.#{unique_key}; + ELSIF (TG_OP = 'UPDATE') THEN + UPDATE #{target_table} + SET #{set_statements.join(delimiter)} + WHERE #{target_table}.#{unique_key} = NEW.#{unique_key}; + ELSIF (TG_OP = 'INSERT') THEN + INSERT INTO #{target_table} (#{column_names.join(delimiter)}) + VALUES (#{insert_values.join(delimiter)}); + END IF; + RETURN NULL; + SQL + end + end + + def build_set_statements(column_names, unique_key) + column_names.reject { |name| name == unique_key }.map { |column_name| "#{column_name} = NEW.#{column_name}" } + end + end + end + end +end diff --git a/lib/gitlab/database/schema_cleaner.rb b/lib/gitlab/database/schema_cleaner.rb index c1436d3e7ca..ae9d77e635e 100644 --- a/lib/gitlab/database/schema_cleaner.rb +++ b/lib/gitlab/database/schema_cleaner.rb @@ -12,27 +12,15 @@ module Gitlab def clean(io) structure = original_schema.dup - # Postgres compat fix for PG 9.6 (which doesn't support (AS datatype) syntax for sequences) - structure.gsub!(/CREATE SEQUENCE [^.]+\.\S+\n(\s+AS integer\n)/) { |m| m.gsub(Regexp.last_match[1], '') } - - # Also a PG 9.6 compatibility fix, see below. - structure.gsub!(/^CREATE EXTENSION IF NOT EXISTS plpgsql.*/, '') - structure.gsub!(/^COMMENT ON EXTENSION.*/, '') - # Remove noise + structure.gsub!(/^COMMENT ON EXTENSION.*/, '') structure.gsub!(/^SET.+/, '') structure.gsub!(/^SELECT pg_catalog\.set_config\('search_path'.+/, '') structure.gsub!(/^--.*/, "\n") - structure.gsub!(/\n{3,}/, "\n\n") - io << "SET search_path=public;\n\n" + structure = "SET search_path=public;\n" + structure - # Adding plpgsql explicitly is again a compatibility fix for PG 9.6 - # In more recent versions of pg_dump, the extension isn't explicitly dumped anymore. - # We use PG 9.6 still on CI and for schema checks - here this is still the case. - io << <<~SQL.strip - CREATE EXTENSION IF NOT EXISTS plpgsql WITH SCHEMA pg_catalog; - SQL + structure.gsub!(/\n{3,}/, "\n\n") io << structure diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index f8d01c78ae8..8e544307d81 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -16,12 +16,12 @@ module Gitlab SQL end - def create_function_trigger(name, fn_name, fires: nil) + def create_trigger(name, function_name, fires: nil) execute(<<~SQL) CREATE TRIGGER #{name} #{fires} FOR EACH ROW - EXECUTE PROCEDURE #{fn_name}() + EXECUTE PROCEDURE #{function_name}() SQL end @@ -35,6 +35,16 @@ module Gitlab execute("DROP TRIGGER #{exists_clause} #{name} ON #{table_name}") end + def create_comment(type, name, text) + execute("COMMENT ON #{type} #{name} IS '#{text}'") + end + + def tmp_table_name(base) + hashed_base = Digest::SHA256.hexdigest(base).first(10) + + "#{base}_#{hashed_base}" + end + def object_name(table, type) identifier = "#{table}_#{type}" hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) @@ -42,8 +52,30 @@ module Gitlab "#{type}_#{hashed_identifier}" end + def with_lock_retries(&block) + Gitlab::Database::WithLockRetries.new({ + klass: self.class, + logger: Gitlab::BackgroundMigration::Logger + }).run(&block) + end + + def assert_not_in_transaction_block(scope:) + return unless transaction_open? + + raise "#{scope} operations can not be run inside a transaction block, " \ + "you can disable transaction blocks by calling disable_ddl_transaction! " \ + "in the body of your migration class" + end + private + def create_range_partition(partition_name, table_name, lower_bound, upper_bound) + execute(<<~SQL) + CREATE TABLE #{partition_name} PARTITION OF #{table_name} + FOR VALUES FROM (#{lower_bound}) TO (#{upper_bound}) + SQL + end + def optional_clause(flag, clause) flag ? clause : "" end diff --git a/lib/gitlab/dependency_linker.rb b/lib/gitlab/dependency_linker.rb index 7af380689d5..db60128b979 100644 --- a/lib/gitlab/dependency_linker.rb +++ b/lib/gitlab/dependency_linker.rb @@ -13,7 +13,9 @@ module Gitlab CartfileLinker, GodepsJsonLinker, RequirementsTxtLinker, - CargoTomlLinker + CargoTomlLinker, + GoModLinker, + GoSumLinker ].freeze def self.linker(blob_name) diff --git a/lib/gitlab/dependency_linker/go_mod_linker.rb b/lib/gitlab/dependency_linker/go_mod_linker.rb new file mode 100644 index 00000000000..4d6fe366333 --- /dev/null +++ b/lib/gitlab/dependency_linker/go_mod_linker.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module DependencyLinker + class GoModLinker < BaseLinker + include Gitlab::Golang + + self.file_type = :go_mod + + private + + SEMVER = Gitlab::Regex.unbounded_semver_regex + NAME = Gitlab::Regex.go_package_regex + REGEX = Regexp.new("(?<name>#{NAME.source})(?:\\s+(?<version>v#{SEMVER.source}))?", SEMVER.options | NAME.options).freeze + + # rubocop: disable CodeReuse/ActiveRecord + def link_dependencies + highlighted_lines.map!.with_index do |rich_line, i| + plain_line = plain_lines[i].chomp + match = REGEX.match(plain_line) + next rich_line unless match + + i, j = match.offset(:name) + marker = StringRangeMarker.new(plain_line, rich_line.html_safe) + marker.mark([i..(j - 1)]) do |text, left:, right:| + url = package_url(text, match[:version]) + url ? link_tag(text, url) : text + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/dependency_linker/go_sum_linker.rb b/lib/gitlab/dependency_linker/go_sum_linker.rb new file mode 100644 index 00000000000..20dc82ede9f --- /dev/null +++ b/lib/gitlab/dependency_linker/go_sum_linker.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module DependencyLinker + class GoSumLinker < GoModLinker + self.file_type = :go_sum + + private + + BASE64 = Gitlab::Regex.base64_regex + REGEX = Regexp.new("^\\s*(?<name>#{NAME.source})\\s+(?<version>v#{SEMVER.source})(\/go.mod)?\\s+h1:(?<checksum>#{BASE64.source})\\s*$", NAME.options).freeze + + # rubocop: disable CodeReuse/ActiveRecord + def link_dependencies + highlighted_lines.map!.with_index do |rich_line, i| + plain_line = plain_lines[i].chomp + match = REGEX.match(plain_line) + next rich_line unless match + + i0, j0 = match.offset(:name) + i2, j2 = match.offset(:checksum) + + marker = StringRangeMarker.new(plain_line, rich_line.html_safe) + marker.mark([i0..(j0 - 1), i2..(j2 - 1)]) do |text, left:, right:| + if left + url = package_url(text, match[:version]) + url ? link_tag(text, url) : text + + elsif right + link_tag(text, "https://sum.golang.org/lookup/#{match[:name]}@#{match[:version]}") + end + end + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end +end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index d1398ddb642..72dcc4fde71 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -225,6 +225,10 @@ module Gitlab new_path.presence || old_path end + def file_hash + Digest::SHA1.hexdigest(file_path) + end + def added_lines @stats&.additions || diff_lines.count(&:added?) end @@ -237,6 +241,10 @@ module Gitlab "#{file_path}-#{new_file?}-#{deleted_file?}-#{renamed_file?}" end + def file_identifier_hash + Digest::SHA1.hexdigest(file_identifier) + end + def diffable? repository.attributes(file_path).fetch('diff') { true } end diff --git a/lib/gitlab/diff/formatters/base_formatter.rb b/lib/gitlab/diff/formatters/base_formatter.rb index 9704aed82c1..31eeadc45f7 100644 --- a/lib/gitlab/diff/formatters/base_formatter.rb +++ b/lib/gitlab/diff/formatters/base_formatter.rb @@ -6,6 +6,7 @@ module Gitlab class BaseFormatter attr_reader :old_path attr_reader :new_path + attr_reader :file_identifier_hash attr_reader :base_sha attr_reader :start_sha attr_reader :head_sha @@ -16,6 +17,7 @@ module Gitlab attrs[:diff_refs] = diff_file.diff_refs attrs[:old_path] = diff_file.old_path attrs[:new_path] = diff_file.new_path + attrs[:file_identifier_hash] = diff_file.file_identifier_hash end if diff_refs = attrs[:diff_refs] @@ -26,6 +28,7 @@ module Gitlab @old_path = attrs[:old_path] @new_path = attrs[:new_path] + @file_identifier_hash = attrs[:file_identifier_hash] @base_sha = attrs[:base_sha] @start_sha = attrs[:start_sha] @head_sha = attrs[:head_sha] @@ -36,7 +39,7 @@ module Gitlab end def to_h - { + out = { base_sha: base_sha, start_sha: start_sha, head_sha: head_sha, @@ -44,6 +47,12 @@ module Gitlab new_path: new_path, position_type: position_type } + + if Feature.enabled?(:file_identifier_hash) + out[:file_identifier_hash] = file_identifier_hash + end + + out end def position_type diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index 10ad23b7774..e43f301c280 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -9,6 +9,7 @@ module Gitlab delegate :old_path, :new_path, + :file_identifier_hash, :base_sha, :start_sha, :head_sha, @@ -156,13 +157,23 @@ module Gitlab position_type == 'text' end + def find_diff_file_from(diffable) + diff_files = diffable.diffs(diff_options).diff_files + + if Feature.enabled?(:file_identifier_hash) && file_identifier_hash.present? + diff_files.find { |df| df.file_identifier_hash == file_identifier_hash } + else + diff_files.first + end + end + private def find_diff_file(repository) return unless diff_refs.complete? return unless comparison = diff_refs.compare_in(repository.project) - comparison.diffs(diff_options).diff_files.first + find_diff_file_from(comparison) end def get_formatter_class(type) diff --git a/lib/gitlab/doctor/secrets.rb b/lib/gitlab/doctor/secrets.rb new file mode 100644 index 00000000000..31c5dded3ff --- /dev/null +++ b/lib/gitlab/doctor/secrets.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module Doctor + class Secrets + attr_reader :logger + + def initialize(logger) + @logger = logger + end + + def run! + logger.info "Checking encrypted values in the database" + Rails.application.eager_load! unless Rails.application.config.eager_load + + models_with_attributes = Hash.new { |h, k| h[k] = [] } + + models_with_encrypted_attributes.each do |model| + models_with_attributes[model] += model.encrypted_attributes.keys + end + + models_with_encrypted_tokens.each do |model| + models_with_attributes[model] += model.encrypted_token_authenticatable_fields + end + + check_model_attributes(models_with_attributes) + + logger.info "Done!" + end + + private + + def check_model_attributes(models_with_attributes) + running_failures = 0 + + models_with_attributes.each do |model, attributes| + failures_per_row = Hash.new { |h, k| h[k] = [] } + model.find_each do |data| + attributes.each do |att| + failures_per_row[data.id] << att unless valid_attribute?(data, att) + end + end + + running_failures += failures_per_row.keys.count + output_failures_for_model(model, failures_per_row) + end + + logger.info "Total: #{running_failures} row(s) affected".color(:blue) + end + + def output_failures_for_model(model, failures) + status_color = failures.empty? ? :green : :red + + logger.info "- #{model} failures: #{failures.count}".color(status_color) + failures.each do |row_id, attributes| + logger.debug " - #{model}[#{row_id}]: #{attributes.join(", ")}".color(:red) + end + end + + def models_with_encrypted_attributes + all_models.select { |d| d.encrypted_attributes.present? } + end + + def models_with_encrypted_tokens + all_models.select do |d| + d.include?(TokenAuthenticatable) && d.encrypted_token_authenticatable_fields.present? + end + end + + def all_models + @all_models ||= ApplicationRecord.descendants + end + + def valid_attribute?(data, attr) + data.public_send(attr) # rubocop:disable GitlabSecurity/PublicSend + + true + rescue OpenSSL::Cipher::CipherError, TypeError + false + rescue => e + logger.debug "> Something went wrong for #{data.class.name}[#{data.id}].#{attr}: #{e}".color(:red) + + false + end + end + end +end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index b893d625f8d..a19ce22e53f 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -26,10 +26,13 @@ module Gitlab # Sanitize fields based on those sanitized from Rails. config.sanitize_fields = Rails.application.config.filter_parameters.map(&:to_s) + config.processors << ::Gitlab::ErrorTracking::Processor::SidekiqProcessor # Sanitize authentication headers config.sanitize_http_headers = %w[Authorization Private-Token] config.tags = { program: Gitlab.process_name } config.before_send = method(:before_send) + + yield config if block_given? end end diff --git a/lib/gitlab/error_tracking/processor/sidekiq_processor.rb b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb new file mode 100644 index 00000000000..272cb689ad5 --- /dev/null +++ b/lib/gitlab/error_tracking/processor/sidekiq_processor.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'set' + +module Gitlab + module ErrorTracking + module Processor + class SidekiqProcessor < ::Raven::Processor + FILTERED_STRING = '[FILTERED]' + + def self.filter_arguments(args, klass) + args.lazy.with_index.map do |arg, i| + case arg + when Numeric + arg + else + if permitted_arguments_for_worker(klass).include?(i) + arg + else + FILTERED_STRING + end + end + end + end + + def self.permitted_arguments_for_worker(klass) + @permitted_arguments_for_worker ||= {} + @permitted_arguments_for_worker[klass] ||= + begin + klass.constantize&.loggable_arguments&.to_set + rescue + Set.new + end + end + + def self.loggable_arguments(args, klass) + Gitlab::Utils::LogLimitedArray + .log_limited_array(filter_arguments(args, klass)) + .map(&:to_s) + .to_a + end + + def process(value, key = nil) + sidekiq = value.dig(:extra, :sidekiq) + + return value unless sidekiq + + sidekiq = sidekiq.deep_dup + sidekiq.delete(:jobstr) + + # 'args' in this hash => from Gitlab::ErrorTracking.track_* + # 'args' in :job => from default error handler + job_holder = sidekiq.key?('args') ? sidekiq : sidekiq[:job] + + if job_holder['args'] + job_holder['args'] = self.class.filter_arguments(job_holder['args'], job_holder['class']).to_a + end + + value[:extra][:sidekiq] = sidekiq + + value + end + end + end + end +end diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb index 92d55213cc2..2da1b8915e4 100644 --- a/lib/gitlab/exception_log_formatter.rb +++ b/lib/gitlab/exception_log_formatter.rb @@ -12,6 +12,16 @@ module Gitlab 'exception.message' => exception.message ) + payload.delete('extra.server') + + payload['extra.sidekiq'].tap do |value| + if value.is_a?(Hash) && value.key?('args') + value = value.dup + payload['extra.sidekiq']['args'] = Gitlab::ErrorTracking::Processor::SidekiqProcessor + .loggable_arguments(value['args'], value['class']) + end + end + if exception.backtrace payload['exception.backtrace'] = Gitlab::BacktraceCleaner.clean_backtrace(exception.backtrace) end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 3495b4a0b72..d3df9be0d63 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -13,18 +13,20 @@ # To enable the experiment for 10% of the users: # # chatops: `/chatops run feature set experiment_key_experiment_percentage 10` -# console: `Feature.get(:experiment_key_experiment_percentage).enable_percentage_of_time(10)` +# console: `Feature.enable_percentage_of_time(:experiment_key_experiment_percentage, 10)` # # To disable the experiment: # # chatops: `/chatops run feature delete experiment_key_experiment_percentage` -# console: `Feature.get(:experiment_key_experiment_percentage).remove` +# console: `Feature.remove(:experiment_key_experiment_percentage)` # # To check the current rollout percentage: # # chatops: `/chatops run feature get experiment_key_experiment_percentage` # console: `Feature.get(:experiment_key_experiment_percentage).percentage_of_time_value` # + +# TODO: see https://gitlab.com/gitlab-org/gitlab/-/issues/217490 module Gitlab module Experimentation EXPERIMENTS = { @@ -45,6 +47,12 @@ module Gitlab }, upgrade_link_in_user_menu_a: { tracking_category: 'Growth::Expansion::Experiment::UpgradeLinkInUserMenuA' + }, + invite_members_version_a: { + tracking_category: 'Growth::Expansion::Experiment::InviteMembersVersionA' + }, + new_create_project_ui: { + tracking_category: 'Manage::Import::Experiment::NewCreateProjectUi' } }.freeze @@ -66,7 +74,6 @@ module Gitlab cookies.permanent.signed[:experimentation_subject_id] = { value: SecureRandom.uuid, - domain: :all, secure: ::Gitlab.config.gitlab.https, httponly: true } @@ -179,7 +186,7 @@ module Gitlab # When a feature does not exist, the `percentage_of_time_value` method will return 0 def experiment_percentage - @experiment_percentage ||= Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value + @experiment_percentage ||= Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value # rubocop:disable Gitlab/AvoidFeatureGet end end end diff --git a/lib/gitlab/export/logger.rb b/lib/gitlab/export/logger.rb new file mode 100644 index 00000000000..b3c05651cd4 --- /dev/null +++ b/lib/gitlab/export/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Export + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'exporter' + end + end + end +end diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index a9e9261cd3c..13959f6aa68 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -32,6 +32,8 @@ module Gitlab gemfile_lock: 'Gemfile.lock', gemspec: %r{\A[^/]*\.gemspec\z}, godeps_json: 'Godeps.json', + go_mod: 'go.mod', + go_sum: 'go.sum', package_json: 'package.json', podfile: 'Podfile', podspec_json: %r{\A[^/]*\.podspec\.json\z}, diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 23af0a9bb18..08321d5fda6 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -22,9 +22,10 @@ module Gitlab return @text unless needs_rewrite? @text.gsub(@pattern) do |markdown| - Gitlab::Utils.check_path_traversal!($~[:file]) + file = find_file($~[:secret], $~[:file]) + # No file will be returned for a path traversal + next if file.nil? - file = find_file(@source_project, $~[:secret], $~[:file]) break markdown unless file.try(:exists?) klass = target_parent.is_a?(Namespace) ? NamespaceFileUploader : FileUploader @@ -47,7 +48,7 @@ module Gitlab def files referenced_files = @text.scan(@pattern).map do - find_file(@source_project, $~[:secret], $~[:file]) + find_file($~[:secret], $~[:file]) end referenced_files.compact.select(&:exists?) @@ -57,12 +58,8 @@ module Gitlab markdown.starts_with?("!") end - private - - def find_file(project, secret, file) - uploader = FileUploader.new(project, secret: secret) - uploader.retrieve_from_store!(file) - uploader + def find_file(secret, file_name) + UploaderFinder.new(@source_project, secret, file_name).execute end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index a554dc0b667..17d0a62ba8c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -7,6 +7,7 @@ module Gitlab include Gitlab::EncodingHelper prepend Gitlab::Git::RuggedImpl::Commit extend Gitlab::Git::WrapsGitalyErrors + include Gitlab::Utils::StrongMemoize attr_accessor :raw_commit, :head @@ -231,6 +232,18 @@ module Gitlab parent_ids.first end + def committed_date + strong_memoize(:committed_date) do + init_date_from_gitaly(raw_commit.committer) if raw_commit + end + end + + def authored_date + strong_memoize(:authored_date) do + init_date_from_gitaly(raw_commit.author) if raw_commit + end + end + # Returns a diff object for the changes from this commit's first parent. # If there is no parent, then the diff is between this commit and an # empty repo. See Repository#diff for keys allowed in the +options+ @@ -369,11 +382,9 @@ module Gitlab # subject from the message to make it clearer when there's one # available but not the other. @message = message_from_gitaly_body - @authored_date = init_date_from_gitaly(commit.author) @author_name = commit.author.name.dup @author_email = commit.author.email.dup - @committed_date = init_date_from_gitaly(commit.committer) @committer_name = commit.committer.name.dup @committer_email = commit.committer.email.dup @parent_ids = Array(commit.parent_ids) diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 74a4633424f..bb845f11181 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -225,7 +225,7 @@ module Gitlab end def init_from_gitaly(diff) - @diff = encode!(diff.patch) if diff.respond_to?(:patch) + @diff = diff.respond_to?(:patch) ? encode!(diff.patch) : '' @new_path = encode!(diff.to_path.dup) @old_path = encode!(diff.from_path.dup) @a_mode = diff.old_mode.to_s(8) diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb index f9573bedba7..dae208e6955 100644 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ b/lib/gitlab/git/rugged_impl/use_rugged.rb @@ -5,8 +5,7 @@ module Gitlab module RuggedImpl module UseRugged def use_rugged?(repo, feature_key) - feature = Feature.get(feature_key) - return feature.enabled? if Feature.persisted?(feature) + return Feature.enabled?(feature_key) if Feature.persisted_name?(feature_key) # Disable Rugged auto-detect(can_use_disk?) when Puma threads>1 # https://gitlab.com/gitlab-org/gitlab/issues/119326 @@ -25,7 +24,7 @@ module Gitlab if Gitlab::RuggedInstrumentation.active? Gitlab::RuggedInstrumentation.increment_query_count - Gitlab::RuggedInstrumentation.query_time += duration + Gitlab::RuggedInstrumentation.add_query_time(duration) Gitlab::RuggedInstrumentation.add_call_details( feature: method_name, diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index f1ca8900c30..37e3da984d6 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -8,7 +8,6 @@ module Gitlab ForbiddenError = Class.new(StandardError) NotFoundError = Class.new(StandardError) - ProjectCreationError = Class.new(StandardError) TimeoutError = Class.new(StandardError) ProjectMovedError = Class.new(NotFoundError) @@ -24,6 +23,7 @@ module Gitlab deploy_key_upload: 'This deploy key does not have write access to this project.', no_repo: 'A repository for this project does not exist yet.', project_not_found: 'The project you were looking for could not be found.', + namespace_not_found: 'The namespace you were looking for could not be found.', command_not_allowed: "The command you're trying to execute is not allowed.", upload_pack_disabled_over_http: 'Pulling over HTTP is not allowed.', receive_pack_disabled_over_http: 'Pushing over HTTP is not allowed.', @@ -73,7 +73,8 @@ module Gitlab return custom_action if custom_action check_db_accessibility!(cmd) - check_project!(changes, cmd) + check_namespace! + check_project!(cmd) check_repository_existence! case cmd @@ -110,9 +111,7 @@ module Gitlab private - def check_project!(changes, cmd) - check_namespace! - ensure_project_on_push!(cmd, changes) + def check_project!(_cmd) check_project_accessibility! add_project_moved_message! end @@ -156,7 +155,7 @@ module Gitlab def check_namespace! return if namespace_path.present? - raise NotFoundError, ERROR_MESSAGES[:project_not_found] + raise NotFoundError, ERROR_MESSAGES[:namespace_not_found] end def check_active_user! @@ -229,32 +228,6 @@ module Gitlab end end - def ensure_project_on_push!(cmd, changes) - return if project || deploy_key? - return unless receive_pack?(cmd) && changes == ANY && authentication_abilities.include?(:push_code) - - namespace = Namespace.find_by_full_path(namespace_path) - - return unless user&.can?(:create_projects, namespace) - - project_params = { - path: repository_path, - namespace_id: namespace.id, - visibility_level: Gitlab::VisibilityLevel::PRIVATE - } - - project = Projects::CreateService.new(user, project_params).execute - - unless project.saved? - raise ProjectCreationError, "Could not create project: #{project.errors.full_messages.join(', ')}" - end - - @project = project - user_access.project = @project - - Checks::ProjectCreated.new(repository, user, protocol).add_message - end - def check_repository_existence! unless repository.exists? raise NotFoundError, ERROR_MESSAGES[:no_repo] diff --git a/lib/gitlab/git_access_project.rb b/lib/gitlab/git_access_project.rb new file mode 100644 index 00000000000..c79a61c263e --- /dev/null +++ b/lib/gitlab/git_access_project.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + class GitAccessProject < GitAccess + extend ::Gitlab::Utils::Override + + CreationError = Class.new(StandardError) + + private + + override :check_project! + def check_project!(cmd) + ensure_project_on_push!(cmd) + + super + end + + def ensure_project_on_push!(cmd) + return if project || deploy_key? + return unless receive_pack?(cmd) && changes == ANY && authentication_abilities.include?(:push_code) + + namespace = Namespace.find_by_full_path(namespace_path) + + return unless user&.can?(:create_projects, namespace) + + project_params = { + path: repository_path, + namespace_id: namespace.id, + visibility_level: Gitlab::VisibilityLevel::PRIVATE + } + + project = Projects::CreateService.new(user, project_params).execute + + unless project.saved? + raise CreationError, "Could not create project: #{project.errors.full_messages.join(', ')}" + end + + @project = project + user_access.project = @project + + Checks::ProjectCreated.new(repository, user, protocol).add_message + end + end +end diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index 70db4271469..3de6c9ee30a 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -39,13 +39,18 @@ module Gitlab private + override :check_namespace! + def check_namespace! + return unless snippet.is_a?(ProjectSnippet) + + super + end + override :check_project! - def check_project!(cmd, changes) + def check_project!(cmd) return unless snippet.is_a?(ProjectSnippet) - check_namespace! - check_project_accessibility! - add_project_moved_message! + super end override :check_push_access! diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb index 3c0dbba64bf..aad46937c32 100644 --- a/lib/gitlab/git_access_wiki.rb +++ b/lib/gitlab/git_access_wiki.rb @@ -2,6 +2,8 @@ module Gitlab class GitAccessWiki < GitAccess + prepend_if_ee('EE::Gitlab::GitAccessWiki') # rubocop: disable Cop/InjectEnterpriseEditionModule + ERROR_MESSAGES = { read_only: "You can't push code to a read-only GitLab instance.", write_to_wiki: "You are not allowed to write to this project's wiki." @@ -31,8 +33,10 @@ module Gitlab ERROR_MESSAGES[:read_only] end - def container - project.wiki + private + + def repository + project.wiki.repository end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 3aaed0edb87..bed99ef0ed4 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -201,7 +201,8 @@ module Gitlab request_hash = request.is_a?(Google::Protobuf::MessageExts) ? request.to_h : {} # Keep track, separately, for the performance bar - self.query_time += duration + self.add_query_time(duration) + if Gitlab::PerformanceBar.enabled_for_request? add_call_details(feature: "#{service}##{rpc}", duration: duration, request: request_hash, rpc: rpc, backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller)) @@ -209,12 +210,15 @@ module Gitlab end def self.query_time - query_time = SafeRequestStore[:gitaly_query_time] ||= 0 + query_time = Gitlab::SafeRequestStore[:gitaly_query_time] || 0 query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION) end - def self.query_time=(duration) - SafeRequestStore[:gitaly_query_time] = duration + def self.add_query_time(duration) + return unless Gitlab::SafeRequestStore.active? + + Gitlab::SafeRequestStore[:gitaly_query_time] ||= 0 + Gitlab::SafeRequestStore[:gitaly_query_time] += duration end def self.current_transaction_labels diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 1f914dc95d1..aed132aaca0 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -212,8 +212,9 @@ module Gitlab right_commit_id: right_commit_sha ) - response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) - response.flat_map(&:stats) + GitalyClient.streaming_call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) do |response| + response.flat_map(&:stats) + end end def find_all_commits(opts = {}) @@ -246,8 +247,8 @@ module Gitlab request = Gitaly::CommitsByMessageRequest.new( repository: @gitaly_repo, query: query, - revision: revision.to_s.force_encoding(Encoding::ASCII_8BIT), - path: path.to_s.force_encoding(Encoding::ASCII_8BIT), + revision: encode_binary(revision), + path: encode_binary(path), limit: limit.to_i, offset: offset.to_i ) diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb index 147597289cf..0d448b55104 100644 --- a/lib/gitlab/github_import/bulk_importing.rb +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -17,7 +17,7 @@ module Gitlab # Bulk inserts the given rows into the database. def bulk_insert(model, rows, batch_size: 100) rows.each_slice(batch_size) do |slice| - Gitlab::Database.bulk_insert(model.table_name, slice) + Gitlab::Database.bulk_insert(model.table_name, slice) # rubocop:disable Gitlab/BulkInsert end end end diff --git a/lib/gitlab/github_import/importer/diff_note_importer.rb b/lib/gitlab/github_import/importer/diff_note_importer.rb index d562958e955..53b17f77ccd 100644 --- a/lib/gitlab/github_import/importer/diff_note_importer.rb +++ b/lib/gitlab/github_import/importer/diff_note_importer.rb @@ -47,7 +47,7 @@ module Gitlab # To work around this we're using bulk_insert with a single row. This # allows us to efficiently insert data (even if it's just 1 row) # without having to use all sorts of hacks to disable callbacks. - Gitlab::Database.bulk_insert(LegacyDiffNote.table_name, [attributes]) + Gitlab::Database.bulk_insert(LegacyDiffNote.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert rescue ActiveRecord::InvalidForeignKey # It's possible the project and the issue have been deleted since # scheduling this job. In this case we'll just skip creating the note. diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index 8648cbaec9d..13061d2c9df 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -75,7 +75,7 @@ module Gitlab end end - Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignees) + Gitlab::Database.bulk_insert(IssueAssignee.table_name, assignees) # rubocop:disable Gitlab/BulkInsert end end end diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb index 2001b7e3482..77eb4542195 100644 --- a/lib/gitlab/github_import/importer/label_links_importer.rb +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -40,7 +40,7 @@ module Gitlab } end - Gitlab::Database.bulk_insert(LabelLink.table_name, rows) + Gitlab::Database.bulk_insert(LabelLink.table_name, rows) # rubocop:disable Gitlab/BulkInsert end def find_target_id diff --git a/lib/gitlab/github_import/importer/lfs_objects_importer.rb b/lib/gitlab/github_import/importer/lfs_objects_importer.rb index 30763492235..5980b3c2179 100644 --- a/lib/gitlab/github_import/importer/lfs_objects_importer.rb +++ b/lib/gitlab/github_import/importer/lfs_objects_importer.rb @@ -29,7 +29,10 @@ module Gitlab yield object end rescue StandardError => e - Rails.logger.error("The Lfs import process failed. #{e.message}") # rubocop:disable Gitlab/RailsLogger + Gitlab::Import::Logger.error( + message: 'The Lfs import process failed', + error: e.message + ) end end end diff --git a/lib/gitlab/github_import/importer/note_importer.rb b/lib/gitlab/github_import/importer/note_importer.rb index 2b06d1b3baf..41f179d275b 100644 --- a/lib/gitlab/github_import/importer/note_importer.rb +++ b/lib/gitlab/github_import/importer/note_importer.rb @@ -38,7 +38,7 @@ module Gitlab # We're using bulk_insert here so we can bypass any validations and # callbacks. Running these would result in a lot of unnecessary SQL # queries being executed when importing large projects. - Gitlab::Database.bulk_insert(Note.table_name, [attributes]) + Gitlab::Database.bulk_insert(Note.table_name, [attributes]) # rubocop:disable Gitlab/BulkInsert rescue ActiveRecord::InvalidForeignKey # It's possible the project and the issue have been deleted since # scheduling this job. In this case we'll just skip creating the note. diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 929fceaacf2..dcae8ca01fa 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -40,8 +40,10 @@ module Gitlab pname = project.path_with_namespace - Rails.logger # rubocop:disable Gitlab/RailsLogger - .info("GitHub importer finished updating repository for #{pname}") + Gitlab::Import::Logger.info( + message: 'GitHub importer finished updating repository', + project_name: pname + ) repository_updates_counter.increment end diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index 8abefad1ef3..abd4e847a50 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -6,20 +6,21 @@ module Gitlab PROJECT = RepoType.new( name: :project, - access_checker_class: Gitlab::GitAccess, + access_checker_class: Gitlab::GitAccessProject, repository_resolver: -> (project) { project&.repository } ).freeze WIKI = RepoType.new( name: :wiki, access_checker_class: Gitlab::GitAccessWiki, - repository_resolver: -> (project) { project&.wiki&.repository }, + repository_resolver: -> (container) { container&.wiki&.repository }, + project_resolver: -> (container) { container.is_a?(Project) ? container : nil }, suffix: :wiki ).freeze SNIPPET = RepoType.new( name: :snippet, access_checker_class: Gitlab::GitAccessSnippet, repository_resolver: -> (snippet) { snippet&.repository }, - container_resolver: -> (id) { Snippet.find_by_id(id) }, + container_class: Snippet, project_resolver: -> (snippet) { snippet&.project }, guest_read_ability: :read_snippet ).freeze @@ -42,16 +43,12 @@ module Gitlab end def self.parse(gl_repository) - type_name, _id = gl_repository.split('-').first - type = types[type_name] + result = ::Gitlab::GlRepository::Identifier.new(gl_repository) - unless type - raise ArgumentError, "Invalid GL Repository \"#{gl_repository}\"" - end + repo_type = result.repo_type + container = result.fetch_container! - container = type.fetch_container!(gl_repository) - - [container, type.project_for(container), type] + [container, repo_type.project_for(container), repo_type] end def self.default_type diff --git a/lib/gitlab/gl_repository/identifier.rb b/lib/gitlab/gl_repository/identifier.rb new file mode 100644 index 00000000000..dc3e7931696 --- /dev/null +++ b/lib/gitlab/gl_repository/identifier.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + class GlRepository + class Identifier + attr_reader :gl_repository, :repo_type + + def initialize(gl_repository) + @gl_repository = gl_repository + @segments = gl_repository.split('-') + + raise_error if segments.size > 3 + + @repo_type = find_repo_type + @container_id = find_container_id + @container_class = find_container_class + end + + def fetch_container! + container_class.find_by_id(container_id) + end + + private + + attr_reader :segments, :container_class, :container_id + + def find_repo_type + type_name = three_segments_format? ? segments.last : segments.first + type = Gitlab::GlRepository.types[type_name] + + raise_error unless type + + type + end + + def find_container_class + if three_segments_format? + case segments[0] + when 'project' + Project + when 'group' + Group + else + raise_error + end + else + repo_type.container_class + end + end + + def find_container_id + id = Integer(segments[1], 10, exception: false) + + raise_error unless id + + id + end + + # gl_repository can either have 2 or 3 segments: + # "wiki-1" is the older 2-segment format, where container is implied. + # "group-1-wiki" is the newer 3-segment format, including container information. + # + # TODO: convert all 2-segment format to 3-segment: + # https://gitlab.com/gitlab-org/gitlab/-/issues/219192 + def three_segments_format? + segments.size == 3 + end + + def raise_error + raise ArgumentError, "Invalid GL Repository \"#{gl_repository}\"" + end + end + end +end diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 64c329b15ae..2b482ee3d2d 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -6,7 +6,7 @@ module Gitlab attr_reader :name, :access_checker_class, :repository_resolver, - :container_resolver, + :container_class, :project_resolver, :guest_read_ability, :suffix @@ -15,34 +15,25 @@ module Gitlab name:, access_checker_class:, repository_resolver:, - container_resolver: default_container_resolver, + container_class: default_container_class, project_resolver: nil, guest_read_ability: :download_code, suffix: nil) @name = name @access_checker_class = access_checker_class @repository_resolver = repository_resolver - @container_resolver = container_resolver + @container_class = container_class @project_resolver = project_resolver @guest_read_ability = guest_read_ability @suffix = suffix end def identifier_for_container(container) - "#{name}-#{container.id}" - end - - def fetch_id(identifier) - match = /\A#{name}-(?<id>\d+)\z/.match(identifier) - match[:id] if match - end + if container.is_a?(Group) + return "#{container.class.name.underscore}-#{container.id}-#{name}" + end - def fetch_container!(identifier) - id = fetch_id(identifier) - - raise ArgumentError, "Invalid GL Repository \"#{identifier}\"" unless id - - container_resolver.call(id) + "#{name}-#{container.id}" end def wiki? @@ -85,8 +76,8 @@ module Gitlab private - def default_container_resolver - -> (id) { Project.find_by_id(id) } + def default_container_class + Project end end end diff --git a/lib/gitlab/golang.rb b/lib/gitlab/golang.rb new file mode 100644 index 00000000000..f2dc668c482 --- /dev/null +++ b/lib/gitlab/golang.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module Golang + extend self + + def local_module_prefix + @gitlab_prefix ||= "#{Settings.build_gitlab_go_url}/".freeze + end + + def semver_tag?(tag) + return false if tag.dereferenced_target.nil? + + Packages::SemVer.match?(tag.name, prefixed: true) + end + + def pseudo_version?(version) + return false unless version + + if version.is_a? String + version = parse_semver version + return false unless version + end + + pre = version.prerelease + + # Valid pseudo-versions are: + # vX.0.0-yyyymmddhhmmss-sha1337beef0, when no earlier tagged commit exists for X + # vX.Y.Z-pre.0.yyyymmddhhmmss-sha1337beef0, when most recent prior tag is vX.Y.Z-pre + # vX.Y.(Z+1)-0.yyyymmddhhmmss-sha1337beef0, when most recent prior tag is vX.Y.Z + + if version.minor != 0 || version.patch != 0 + m = /\A(.*\.)?0\./.freeze.match pre + return false unless m + + pre = pre[m[0].length..] + end + + # This pattern is intentionally more forgiving than the patterns + # above. Correctness is verified by #pseudo_version_commit. + /\A\d{14}-\h+\z/.freeze.match? pre + end + + def pseudo_version_commit(project, semver) + # Per Go's implementation of pseudo-versions, a tag should be + # considered a pseudo-version if it matches one of the patterns + # listed in #pseudo_version?, regardless of the content of the + # timestamp or the length of the SHA fragment. However, an error + # should be returned if the timestamp is not correct or if the SHA + # fragment is not exactly 12 characters long. See also Go's + # implementation of: + # + # - [*codeRepo.validatePseudoVersion](https://github.com/golang/go/blob/daf70d6c1688a1ba1699c933b3c3f04d6f2f73d9/src/cmd/go/internal/modfetch/coderepo.go#L530) + # - [Pseudo-version parsing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/pseudo.go) + # - [Pseudo-version request processing](https://github.com/golang/go/blob/master/src/cmd/go/internal/modfetch/coderepo.go) + + # Go ignores anything before '.' or after the second '-', so we will do the same + timestamp, sha = semver.prerelease.split('-').last 2 + timestamp = timestamp.split('.').last + commit = project.repository.commit_by(oid: sha) + + # Error messages are based on the responses of proxy.golang.org + + # Verify that the SHA fragment references a commit + raise ArgumentError.new 'invalid pseudo-version: unknown commit' unless commit + + # Require the SHA fragment to be 12 characters long + raise ArgumentError.new 'invalid pseudo-version: revision is shorter than canonical' unless sha.length == 12 + + # Require the timestamp to match that of the commit + raise ArgumentError.new 'invalid pseudo-version: does not match version-control timestamp' unless commit.committed_date.strftime('%Y%m%d%H%M%S') == timestamp + + commit + end + + def parse_semver(str) + Packages::SemVer.parse(str, prefixed: true) + end + + def pkg_go_dev_url(name, version = nil) + if version + "https://pkg.go.dev/#{name}@#{version}" + else + "https://pkg.go.dev/#{name}" + end + end + + def package_url(name, version = nil) + return unless UrlSanitizer.valid?("https://#{name}") + + return pkg_go_dev_url(name, version) unless name.starts_with?(local_module_prefix) + + # This will not work if `name` refers to a subdirectory of a project. This + # could be expanded with logic similar to Gitlab::Middleware::Go to locate + # the project, check for permissions, and return a smarter result. + "#{Gitlab.config.gitlab.protocol}://#{name}/" + end + end +end diff --git a/lib/gitlab/graphql/authorize/authorize_field_service.rb b/lib/gitlab/graphql/authorize/authorize_field_service.rb index 61668b634fd..cbf3e7b8429 100644 --- a/lib/gitlab/graphql/authorize/authorize_field_service.rb +++ b/lib/gitlab/graphql/authorize/authorize_field_service.rb @@ -84,13 +84,25 @@ module Gitlab elsif resolved_type.is_a? Array # A simple list of rendered types each object being an object to authorize resolved_type.select do |single_object_type| - allowed_access?(current_user, single_object_type.object) + allowed_access?(current_user, realized(single_object_type).object) end else raise "Can't authorize #{@field}" end end + # Ensure that we are dealing with realized objects, not delayed promises + def realized(thing) + case thing + when BatchLoader::GraphQL + thing.sync + when GraphQL::Execution::Lazy + thing.value # part of the private api, but we need to unwrap it here. + else + thing + end + end + def allowed_access?(current_user, object) object = object.sync if object.respond_to?(:sync) diff --git a/lib/gitlab/graphql/authorize/instrumentation.rb b/lib/gitlab/graphql/authorize/instrumentation.rb index f9ff2b30eae..15ecc3b04f0 100644 --- a/lib/gitlab/graphql/authorize/instrumentation.rb +++ b/lib/gitlab/graphql/authorize/instrumentation.rb @@ -9,16 +9,12 @@ module Gitlab def instrument(_type, field) service = AuthorizeFieldService.new(field) - if service.authorizations? && !resolver_skips_authorizations?(field) + if service.authorizations? field.redefine { resolve(service.authorized_resolve) } else field end end - - def resolver_skips_authorizations?(field) - field.metadata[:resolver].try(:skip_authorizations?) - end end end end diff --git a/lib/gitlab/graphql/filterable_array.rb b/lib/gitlab/graphql/filterable_array.rb deleted file mode 100644 index 4909d291fd6..00000000000 --- a/lib/gitlab/graphql/filterable_array.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - class FilterableArray < Array - attr_reader :filter_callback - - def initialize(filter_callback, *args) - super(args) - @filter_callback = filter_callback - end - end - end -end diff --git a/lib/gitlab/graphql/loaders/full_path_model_loader.rb b/lib/gitlab/graphql/loaders/full_path_model_loader.rb new file mode 100644 index 00000000000..0aa237c78de --- /dev/null +++ b/lib/gitlab/graphql/loaders/full_path_model_loader.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + # Suitable for use to find resources that expose `where_full_path_in`, + # such as Project, Group, Namespace + class FullPathModelLoader + attr_reader :model_class, :full_path + + def initialize(model_class, full_path) + @model_class, @full_path = model_class, full_path + end + + def find + BatchLoader::GraphQL.for(full_path).batch(key: model_class) do |full_paths, loader, args| + # `with_route` avoids an N+1 calculating full_path + args[:key].where_full_path_in(full_paths).with_route.each do |model_instance| + loader.call(model_instance.full_path, model_instance) + end + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/connections.rb b/lib/gitlab/graphql/pagination/connections.rb index febdc938317..8f37fa3f474 100644 --- a/lib/gitlab/graphql/pagination/connections.rb +++ b/lib/gitlab/graphql/pagination/connections.rb @@ -10,10 +10,6 @@ module Gitlab Gitlab::Graphql::Pagination::Keyset::Connection) schema.connections.add( - Gitlab::Graphql::FilterableArray, - Gitlab::Graphql::Pagination::FilterableArrayConnection) - - schema.connections.add( Gitlab::Graphql::ExternallyPaginatedArray, Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection) end diff --git a/lib/gitlab/graphql/pagination/filterable_array_connection.rb b/lib/gitlab/graphql/pagination/filterable_array_connection.rb deleted file mode 100644 index 4a76cd5fb00..00000000000 --- a/lib/gitlab/graphql/pagination/filterable_array_connection.rb +++ /dev/null @@ -1,17 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module Pagination - # FilterableArrayConnection is useful especially for lazy-loaded values. - # It allows us to call a callback only on the slice of array being - # rendered in the "after loaded" phase. For example we can check - # permissions only on a small subset of items. - class FilterableArrayConnection < GraphQL::Pagination::ArrayConnection - def nodes - @nodes ||= items.filter_callback.call(super) - end - end - end - end -end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index 1a32ab468b1..17cd22d38ad 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -32,6 +32,49 @@ module Gitlab class Connection < GraphQL::Pagination::ActiveRecordRelationConnection include Gitlab::Utils::StrongMemoize + # rubocop: disable Naming/PredicateName + # https://relay.dev/graphql/connections.htm#sec-undefined.PageInfo.Fields + def has_previous_page + strong_memoize(:has_previous_page) do + if after + # If `after` is specified, that points to a specific record, + # even if it's the first one. Since we're asking for `after`, + # then the specific record we're pointing to is in the + # previous page + true + elsif last + limited_nodes + !!@has_previous_page + else + # Key thing to remember. When `before` is specified (and no `last`), + # the spec says return _all_ edges minus anything after the `before`. + # Which means the returned list starts at the very first record. + # Then the max_page kicks in, and returns the first max_page items. + # Because of this, `has_previous_page` will be false + false + end + end + end + + def has_next_page + strong_memoize(:has_next_page) do + if before + # If `before` is specified, that points to a specific record, + # even if it's the last one. Since we're asking for `before`, + # then the specific record we're pointing to is in the + # next page + true + elsif first + # If we count the number of requested items plus one (`limit_value + 1`), + # then if we get `limit_value + 1` then we know there is a next page + relation_count(set_limit(sliced_nodes, limit_value + 1)) == limit_value + 1 + else + false + end + end + end + # rubocop: enable Naming/PredicateName + def cursor_for(node) encoded_json_from_ordering(node) end @@ -39,7 +82,7 @@ module Gitlab def sliced_nodes @sliced_nodes ||= begin - OrderInfo.validate_ordering(ordered_items, order_list) + OrderInfo.validate_ordering(ordered_items, order_list) unless loaded?(ordered_items) sliced = ordered_items sliced = slice_nodes(sliced, before, :before) if before.present? @@ -54,20 +97,30 @@ module Gitlab # So we're ok loading them into memory here as that's bound to happen # anyway. Having them ready means we can modify the result while # rendering the fields. - @nodes ||= load_paged_nodes.to_a + @nodes ||= limited_nodes.to_a end private - def load_paged_nodes - if first && last - raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both") - end + # Apply `first` and `last` to `sliced_nodes` + def limited_nodes + strong_memoize(:limited_nodes) do + if first && last + raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both") + end - if last - sliced_nodes.last(limit_value) - else - sliced_nodes.limit(limit_value) # rubocop: disable CodeReuse/ActiveRecord + if last + # grab one more than we need + paginated_nodes = sliced_nodes.last(limit_value + 1) + + # there is an extra node, so there is a previous page + @has_previous_page = paginated_nodes.count > limit_value + @has_previous_page ? paginated_nodes.last(limit_value) : paginated_nodes + elsif loaded?(sliced_nodes) + sliced_nodes.take(limit_value) # rubocop: disable CodeReuse/ActiveRecord + else + sliced_nodes.limit(limit_value) # rubocop: disable CodeReuse/ActiveRecord + end end end @@ -82,9 +135,19 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def limit_value + # note: only first _or_ last can be specified, not both @limit_value ||= [first, last, max_page_size].compact.min end + def loaded?(items) + case items + when Array + true + else + items.loaded? + end + end + def ordered_items strong_memoize(:ordered_items) do unless items.primary_key.present? @@ -93,6 +156,16 @@ module Gitlab list = OrderInfo.build_order_list(items) + if loaded?(items) + @order_list = list.presence || [items.primary_key] + + # already sorted, or trivially sorted + next items if list.present? || items.size <= 1 + + pkey = items.primary_key.to_sym + next items.sort_by { |item| item[pkey] }.reverse + end + # ensure there is a primary key ordering if list&.last&.attribute_name != items.primary_key items.order(arel_table[items.primary_key].desc) # rubocop: disable CodeReuse/ActiveRecord @@ -121,7 +194,12 @@ module Gitlab order_list.each do |field| field_name = field.attribute_name - ordering[field_name] = node[field_name].to_s + field_value = node[field_name] + ordering[field_name] = if field_value.is_a?(Time) + field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') + else + field_value.to_s + end end encode(ordering.to_json) diff --git a/lib/gitlab/import/database_helpers.rb b/lib/gitlab/import/database_helpers.rb index aaade39dd62..f8ea7a7adcd 100644 --- a/lib/gitlab/import/database_helpers.rb +++ b/lib/gitlab/import/database_helpers.rb @@ -11,7 +11,7 @@ module Gitlab # We use bulk_insert here so we can bypass any queries executed by # callbacks or validation rules, as doing this wouldn't scale when # importing very large projects. - result = Gitlab::Database + result = Gitlab::Database # rubocop:disable Gitlab/BulkInsert .bulk_insert(relation.table_name, [attributes], return_ids: true) result.first diff --git a/lib/gitlab/import/set_async_jid.rb b/lib/gitlab/import/set_async_jid.rb index 3b11c92fb56..527d84477fe 100644 --- a/lib/gitlab/import/set_async_jid.rb +++ b/lib/gitlab/import/set_async_jid.rb @@ -2,8 +2,8 @@ # The original import JID is the JID of the RepositoryImportWorker job, # which will be removed once that job completes. Reusing that JID could -# result in StuckImportJobsWorker marking the job as stuck before we get -# to running Stage::ImportRepositoryWorker. +# result in Gitlab::Import::StuckProjectImportJobsWorker marking the job +# as stuck before we get to running Stage::ImportRepositoryWorker. # # We work around this by setting the JID to a custom generated one, then # refreshing it in the various stages whenever necessary. @@ -13,8 +13,7 @@ module Gitlab def self.set_jid(import_state) jid = generate_jid(import_state) - Gitlab::SidekiqStatus - .set(jid, StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION) + Gitlab::SidekiqStatus.set(jid, Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION) import_state.update_column(:jid, jid) end diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index bab473741b1..1e98595bb07 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -3,6 +3,8 @@ module Gitlab module ImportExport class AttributesFinder + attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads + def initialize(config:) @tree = config[:tree] || {} @included_attributes = config[:included_attributes] || {} diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb new file mode 100644 index 00000000000..86f51add504 --- /dev/null +++ b/lib/gitlab/import_export/attributes_permitter.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +# AttributesPermitter builds a hash of permitted attributes for +# every model defined in import_export.yml that is used to validate and +# filter out any attributes that are not permitted when doing Project/Group Import +# +# Each model's list includes: +# - attributes defined under included_attributes section +# - associations defined under project/group tree +# - methods defined under methods section +# +# Given the following import_export.yml example: +# ``` +# tree: +# project: +# - labels: +# - :priorities +# included_attributes: +# labels: +# - :title +# - :description +# methods: +# labels: +# - :type +# ``` +# +# Produces a list of permitted attributes: +# ``` +# Gitlab::ImportExport::AttributesPermitter.new.permitted_attributes +# +# => { labels: [:priorities, :title, :description, :type] } +# ``` +# +# Filters out any other attributes from specific relation hash: +# ``` +# Gitlab::ImportExport::AttributesPermitter.new.permit(:labels, {id: 5, type: 'opened', description: 'test', sensitive_attribute: 'my_sensitive_attribute'}) +# +# => {:type=>"opened", :description=>"test"} +# ``` +module Gitlab + module ImportExport + class AttributesPermitter + attr_reader :permitted_attributes + + def initialize(config: ImportExport::Config.new.to_h) + @config = config + @attributes_finder = Gitlab::ImportExport::AttributesFinder.new(config: @config) + @permitted_attributes = {} + + build_permitted_attributes + end + + def permit(relation_name, relation_hash) + permitted_attributes = permitted_attributes_for(relation_name) + + relation_hash.select do |key, _| + permitted_attributes.include?(key) + end + end + + def permitted_attributes_for(relation_name) + @permitted_attributes[relation_name] || [] + end + + private + + def build_permitted_attributes + build_associations + build_attributes + build_methods + end + + # Deep traverse relations tree to build a list of allowed model relations + def build_associations + stack = @attributes_finder.tree.to_a + + while stack.any? + model_name, relations = stack.pop + + if relations.is_a?(Hash) + add_permitted_attributes(model_name, relations.keys) + + stack.concat(relations.to_a) + end + end + + @permitted_attributes + end + + def build_attributes + @attributes_finder.included_attributes.each(&method(:add_permitted_attributes)) + end + + def build_methods + @attributes_finder.methods.each(&method(:add_permitted_attributes)) + end + + def add_permitted_attributes(model_name, attributes) + @permitted_attributes[model_name] ||= [] + + @permitted_attributes[model_name].concat(attributes) if attributes.any? + end + end + end +end diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index b1219384732..7b8689069d8 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -24,8 +24,14 @@ module Gitlab raise Projects::ImportService::Error.new(shared.errors.to_sentence) end rescue => e + # If some exception was raised could mean that the SnippetsRepoRestorer + # was not called. This would leave us with snippets without a repository. + # This is a state we don't want them to be, so we better delete them. + remove_non_migrated_snippets + raise Projects::ImportService::Error.new(e.message) ensure + remove_base_tmp_dir remove_import_file end @@ -148,6 +154,18 @@ module Gitlab ::Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}") end end + + def remove_base_tmp_dir + FileUtils.rm_rf(@shared.base_path) + end + + def remove_non_migrated_snippets + project + .snippets + .left_joins(:snippet_repository) + .where(snippet_repositories: { snippet_id: nil }) + .delete_all + end end end end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 7f55a0a3821..20f9c668b9c 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -7,6 +7,15 @@ module Gitlab include Gitlab::ImportExport::CommandLineUtil BATCH_SIZE = 100 + SMALLER_BATCH_SIZE = 20 + + def self.batch_size(exportable) + if Feature.enabled?(:export_reduce_relation_batch_size, exportable) + SMALLER_BATCH_SIZE + else + BATCH_SIZE + end + end class Raw < String def to_json(*_args) @@ -60,7 +69,7 @@ module Gitlab key_preloads = preloads&.dig(key) records = records.preload(key_preloads) if key_preloads - records.find_each(batch_size: BATCH_SIZE) do |record| + records.find_each(batch_size: batch_size) do |record| items << Raw.new(record.to_json(options)) end end @@ -91,6 +100,10 @@ module Gitlab def preloads relations_schema[:preload] end + + def batch_size + @batch_size ||= self.class.batch_size(@exportable) + end end end end diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb index cf75a2c7fa8..f8b8b74ffd7 100644 --- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb +++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb @@ -7,7 +7,7 @@ module Gitlab def serialize(exportable, relations_tree) Gitlab::ImportExport::FastHashSerializer - .new(exportable, relations_tree) + .new(exportable, relations_tree, batch_size: batch_size(exportable)) .execute end @@ -18,6 +18,12 @@ module Gitlab File.write(File.join(dir_path, filename), tree_json) end + + private + + def batch_size(exportable) + Gitlab::ImportExport::JSON::StreamingSerializer.batch_size(exportable) + end end end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 263c49c509f..31d1f7b48bd 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -49,7 +49,7 @@ module Gitlab def ensure_default_member! return if user_already_member? - @importable.members.destroy_all # rubocop: disable DestroyAll + @importable.members.destroy_all # rubocop: disable Cop/DestroyAll relation_class.create!(user: @user, access_level: highest_access_level, source_id: @importable.id, importing: true) rescue => e diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index f735b9612aa..4643742b607 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -41,7 +41,13 @@ module Gitlab def create_source_branch @project.repository.create_branch(@merge_request.source_branch, @diff_head_sha) rescue => err - Rails.logger.warn("Import/Export warning: Failed to create source branch #{@merge_request.source_branch} => #{@diff_head_sha} for MR #{@merge_request.iid}: #{err}") # rubocop:disable Gitlab/RailsLogger + Gitlab::Import::Logger.warn( + message: 'Import warning: Failed to create source branch', + source_branch: @merge_request.source_branch, + diff_head_sha: @diff_head_sha, + merge_request_iid: @merge_request.iid, + error: err.message + ) end def create_target_branch diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 8851b106ad5..f0b733d7e95 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -312,6 +312,7 @@ excluded_attributes: - :pipeline_schedule_id - :merge_request_id - :external_pull_request_id + - :ci_ref_id stages: - :pipeline_id merge_access_levels: @@ -397,3 +398,4 @@ ee: - protected_environments: - :deploy_access_levels - :service_desk_setting + - :security_setting diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 3123687453f..9e10e7aea13 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -16,6 +16,8 @@ module Gitlab repository.create_from_bundle(path_to_bundle) rescue => e + Repositories::DestroyService.new(repository).execute + shared.error(e) false end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index ae82c380755..e4724659eff 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -11,14 +11,16 @@ module Gitlab def initialize(exportable:, shared:) @exportable = exportable - @shared = shared + @shared = shared end def save if compress_and_save - remove_export_path - - Rails.logger.info("Saved #{@exportable.class} export #{archive_file}") # rubocop:disable Gitlab/RailsLogger + Gitlab::Export::Logger.info( + message: 'Export archive saved', + exportable_class: @exportable.class.to_s, + archive_file: archive_file + ) save_upload else @@ -29,8 +31,7 @@ module Gitlab @shared.error(e) false ensure - remove_archive - remove_export_path + remove_base_tmp_dir end private @@ -39,12 +40,8 @@ module Gitlab tar_czf(archive: archive_file, dir: @shared.export_path) end - def remove_export_path - FileUtils.rm_rf(@shared.export_path) - end - - def remove_archive - FileUtils.rm_rf(@shared.archive_path) + def remove_base_tmp_dir + FileUtils.rm_rf(@shared.base_path) end def archive_file diff --git a/lib/gitlab/import_export/snippet_repo_restorer.rb b/lib/gitlab/import_export/snippet_repo_restorer.rb index b58ea14a3a8..334d13a13ae 100644 --- a/lib/gitlab/import_export/snippet_repo_restorer.rb +++ b/lib/gitlab/import_export/snippet_repo_restorer.rb @@ -5,6 +5,8 @@ module Gitlab class SnippetRepoRestorer < RepoRestorer attr_reader :snippet + SnippetRepositoryError = Class.new(StandardError) + def initialize(snippet:, user:, shared:, path_to_bundle:) @snippet = snippet @user = user @@ -34,14 +36,11 @@ module Gitlab end def create_repository_from_db - snippet.create_repository - - commit_attrs = { - branch_name: 'master', - message: 'Initial commit' - } + Gitlab::BackgroundMigration::BackfillSnippetRepositories.new.perform_by_ids([snippet.id]) - repository.create_file(@user, snippet.file_name, snippet.content, commit_attrs) + unless snippet.reset.snippet_repository + raise SnippetRepositoryError, _("Error creating repository for snippet with id %{snippet_id}") % { snippet_id: snippet.id } + end end end end diff --git a/lib/gitlab/import_export/snippets_repo_restorer.rb b/lib/gitlab/import_export/snippets_repo_restorer.rb index 9ff3e74a6b1..5ab28f8dd83 100644 --- a/lib/gitlab/import_export/snippets_repo_restorer.rb +++ b/lib/gitlab/import_export/snippets_repo_restorer.rb @@ -10,15 +10,13 @@ module Gitlab end def restore - return true unless Dir.exist?(snippets_repo_bundle_path) - - @project.snippets.find_each.all? do |snippet| + @project.snippets.find_each.map do |snippet| Gitlab::ImportExport::SnippetRepoRestorer.new(snippet: snippet, user: @user, shared: @shared, path_to_bundle: snippet_repo_bundle_path(snippet)) .restore - end + end.all?(true) end private diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 86ea7a30e69..4154d4fe775 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -36,7 +36,11 @@ module Gitlab def different_version?(version) Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) rescue => e - Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger + Gitlab::Import::Logger.error( + message: 'Import error', + error: e.message + ) + raise Gitlab::ImportExport::Error.new('Incorrect VERSION format') end end diff --git a/lib/gitlab/instrumentation/elasticsearch_transport.rb b/lib/gitlab/instrumentation/elasticsearch_transport.rb new file mode 100644 index 00000000000..deee0127c0c --- /dev/null +++ b/lib/gitlab/instrumentation/elasticsearch_transport.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +require 'elasticsearch-transport' + +module Gitlab + module Instrumentation + module ElasticsearchTransportInterceptor + def perform_request(*args) + start = Time.now + super + ensure + if ::Gitlab::SafeRequestStore.active? + duration = (Time.now - start) + + ::Gitlab::Instrumentation::ElasticsearchTransport.increment_request_count + ::Gitlab::Instrumentation::ElasticsearchTransport.add_duration(duration) + ::Gitlab::Instrumentation::ElasticsearchTransport.add_call_details(duration, args) + end + end + end + + class ElasticsearchTransport + ELASTICSEARCH_REQUEST_COUNT = :elasticsearch_request_count + ELASTICSEARCH_CALL_DURATION = :elasticsearch_call_duration + ELASTICSEARCH_CALL_DETAILS = :elasticsearch_call_details + + def self.get_request_count + ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] || 0 + end + + def self.increment_request_count + ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] ||= 0 + ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] += 1 + end + + def self.detail_store + ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DETAILS] ||= [] + end + + def self.query_time + query_time = ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] || 0 + query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) + end + + def self.add_duration(duration) + ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] ||= 0 + ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] += duration + end + + def self.add_call_details(duration, args) + return unless Gitlab::PerformanceBar.enabled_for_request? + + detail_store << { + method: args[0], + path: args[1], + params: args[2], + body: args[3], + duration: duration, + backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller) + } + end + end + end +end + +class ::Elasticsearch::Transport::Client + prepend ::Gitlab::Instrumentation::ElasticsearchTransportInterceptor +end diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index cc99e828251..82b4701872f 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -1,67 +1,46 @@ # frozen_string_literal: true -require 'redis' - module Gitlab module Instrumentation - module RedisInterceptor - def call(*args, &block) - start = Time.now - super(*args, &block) - ensure - duration = (Time.now - start) - - if ::RequestStore.active? - ::Gitlab::Instrumentation::Redis.increment_request_count - ::Gitlab::Instrumentation::Redis.add_duration(duration) - ::Gitlab::Instrumentation::Redis.add_call_details(duration, args) - end - end - end - + # Aggregates Redis measurements from different request storage sources. class Redis - REDIS_REQUEST_COUNT = :redis_request_count - REDIS_CALL_DURATION = :redis_call_duration - REDIS_CALL_DETAILS = :redis_call_details + ActionCable = Class.new(RedisBase) + Cache = Class.new(RedisBase) + Queues = Class.new(RedisBase) + SharedState = Class.new(RedisBase) - def self.get_request_count - ::RequestStore[REDIS_REQUEST_COUNT] || 0 - end + STORAGES = [ActionCable, Cache, Queues, SharedState].freeze - def self.increment_request_count - ::RequestStore[REDIS_REQUEST_COUNT] ||= 0 - ::RequestStore[REDIS_REQUEST_COUNT] += 1 - end + # Milliseconds represented in seconds (from 1 millisecond to 2 seconds). + QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze - def self.detail_store - ::RequestStore[REDIS_CALL_DETAILS] ||= [] - end + class << self + include ::Gitlab::Instrumentation::RedisPayload - def self.query_time - query_time = ::RequestStore[REDIS_CALL_DURATION] || 0 - query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) - end + def storage_key + nil + end - def self.add_duration(duration) - total_time = query_time + duration - ::RequestStore[REDIS_CALL_DURATION] = total_time - end + def known_payload_keys + super + STORAGES.flat_map(&:known_payload_keys) + end - def self.add_call_details(duration, args) - return unless Gitlab::PerformanceBar.enabled_for_request? - # redis-rb passes an array (e.g. [:get, key]) - return unless args.length == 1 + def payload + super.merge(*STORAGES.flat_map(&:payload)) + end - detail_store << { - cmd: args.first, - duration: duration, - backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller) - } + def detail_store + STORAGES.flat_map do |storage| + storage.detail_store.map { |details| details.merge(storage: storage.name.demodulize) } + end + end + + %i[get_request_count query_time read_bytes write_bytes].each do |method| + define_method method do + STORAGES.sum(&method) # rubocop:disable CodeReuse/ActiveRecord + end + end end end end end - -class ::Redis::Client - prepend ::Gitlab::Instrumentation::RedisInterceptor -end diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb new file mode 100644 index 00000000000..012543e1645 --- /dev/null +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +require 'redis' + +module Gitlab + module Instrumentation + class RedisBase + class << self + include ::Gitlab::Utils::StrongMemoize + include ::Gitlab::Instrumentation::RedisPayload + + # TODO: To be used by https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/395 + # as a 'label' alias. + def storage_key + self.name.demodulize.underscore + end + + def add_duration(duration) + ::RequestStore[call_duration_key] ||= 0 + ::RequestStore[call_duration_key] += duration + end + + def add_call_details(duration, args) + return unless Gitlab::PerformanceBar.enabled_for_request? + # redis-rb passes an array (e.g. [[:get, key]]) + return unless args.length == 1 + + # TODO: Add information about current Redis client + # being instrumented. + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/316. + detail_store << { + cmd: args.first, + duration: duration, + backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller) + } + end + + def increment_request_count + ::RequestStore[request_count_key] ||= 0 + ::RequestStore[request_count_key] += 1 + end + + def increment_read_bytes(num_bytes) + ::RequestStore[read_bytes_key] ||= 0 + ::RequestStore[read_bytes_key] += num_bytes + end + + def increment_write_bytes(num_bytes) + ::RequestStore[write_bytes_key] ||= 0 + ::RequestStore[write_bytes_key] += num_bytes + end + + def get_request_count + ::RequestStore[request_count_key] || 0 + end + + def read_bytes + ::RequestStore[read_bytes_key] || 0 + end + + def write_bytes + ::RequestStore[write_bytes_key] || 0 + end + + def detail_store + ::RequestStore[call_details_key] ||= [] + end + + def query_time + query_time = ::RequestStore[call_duration_key] || 0 + query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) + end + + private + + def request_count_key + strong_memoize(:request_count_key) { build_key(:redis_request_count) } + end + + def read_bytes_key + strong_memoize(:read_bytes_key) { build_key(:redis_read_bytes) } + end + + def write_bytes_key + strong_memoize(:write_bytes_key) { build_key(:redis_write_bytes) } + end + + def call_duration_key + strong_memoize(:call_duration_key) { build_key(:redis_call_duration) } + end + + def call_details_key + strong_memoize(:call_details_key) { build_key(:redis_call_details) } + end + + def build_key(namespace) + "#{storage_key}_#{namespace}" + end + end + end + end +end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb new file mode 100644 index 00000000000..a36aade59c3 --- /dev/null +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +require 'redis' + +module Gitlab + module Instrumentation + module RedisInterceptor + def call(*args, &block) + start = Time.now + super(*args, &block) + ensure + duration = (Time.now - start) + + if ::RequestStore.active? + instrumentation_class.increment_request_count + instrumentation_class.add_duration(duration) + instrumentation_class.add_call_details(duration, args) + end + end + + def write(command) + measure_write_size(command) if ::RequestStore.active? + super + end + + def read + result = super + measure_read_size(result) if ::RequestStore.active? + result + end + + private + + def measure_write_size(command) + size = 0 + + # Mimic what happens in + # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/command_helper.rb#L8. + # This count is an approximation that omits the Redis protocol overhead + # of type prefixes, length prefixes and line endings. + command.each do |x| + size += begin + if x.is_a? Array + x.inject(0) { |sum, y| sum + y.to_s.bytesize } + else + x.to_s.bytesize + end + end + end + + instrumentation_class.increment_write_bytes(size) + end + + def measure_read_size(result) + # The Connection::Ruby#read class can return one of four types of results from read: + # https://github.com/redis/redis-rb/blob/f597f21a6b954b685cf939febbc638f6c803e3a7/lib/redis/connection/ruby.rb#L406 + # + # 1. Error (exception, will not reach this line) + # 2. Status (string) + # 3. Integer (will be converted to string by to_s.bytesize and thrown away) + # 4. "Binary" string (i.e. may contain zero byte) + # 5. Array of binary string + + if result.is_a? Array + # Redis can return nested arrays, e.g. from XRANGE or GEOPOS, so we use recursion here. + result.each { |x| measure_read_size(x) } + else + # This count is an approximation that omits the Redis protocol overhead + # of type prefixes, length prefixes and line endings. + instrumentation_class.increment_read_bytes(result.to_s.bytesize) + end + end + + # That's required so it knows which GitLab Redis instance + # it's interacting with in order to categorize accordingly. + # + def instrumentation_class + @options[:instrumentation_class] # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + end + end +end + +class ::Redis::Client + prepend ::Gitlab::Instrumentation::RedisInterceptor +end diff --git a/lib/gitlab/instrumentation/redis_payload.rb b/lib/gitlab/instrumentation/redis_payload.rb new file mode 100644 index 00000000000..69aafffd124 --- /dev/null +++ b/lib/gitlab/instrumentation/redis_payload.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + module RedisPayload + include ::Gitlab::Utils::StrongMemoize + + # Fetches payload keys from the lazy payload (this avoids + # unnecessary processing of the values). + def known_payload_keys + to_lazy_payload.keys + end + + def payload + to_lazy_payload.transform_values do |value| + result = value.call + result if result > 0 + end.compact + end + + private + + def to_lazy_payload + strong_memoize(:to_lazy_payload) do + key_prefix = storage_key ? "redis_#{storage_key}" : 'redis' + + { + "#{key_prefix}_calls": -> { get_request_count }, + "#{key_prefix}_duration_s": -> { query_time }, + "#{key_prefix}_read_bytes": -> { read_bytes }, + "#{key_prefix}_write_bytes": -> { write_bytes } + }.symbolize_keys + end + end + end + end +end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 7c5a601cd5b..3a29d2e7efa 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -4,30 +4,56 @@ module Gitlab module InstrumentationHelper extend self - KEYS = %i(gitaly_calls gitaly_duration_s rugged_calls rugged_duration_s redis_calls redis_duration_s).freeze DURATION_PRECISION = 6 # microseconds + def keys + @keys ||= [:gitaly_calls, + :gitaly_duration_s, + :rugged_calls, + :rugged_duration_s, + :elasticsearch_calls, + :elasticsearch_duration_s, + *::Gitlab::Instrumentation::Redis.known_payload_keys] + end + def add_instrumentation_data(payload) + instrument_gitaly(payload) + instrument_rugged(payload) + instrument_redis(payload) + instrument_elasticsearch(payload) + end + + def instrument_gitaly(payload) gitaly_calls = Gitlab::GitalyClient.get_request_count - if gitaly_calls > 0 - payload[:gitaly_calls] = gitaly_calls - payload[:gitaly_duration_s] = Gitlab::GitalyClient.query_time - end + return if gitaly_calls == 0 + + payload[:gitaly_calls] = gitaly_calls + payload[:gitaly_duration_s] = Gitlab::GitalyClient.query_time + end + def instrument_rugged(payload) rugged_calls = Gitlab::RuggedInstrumentation.query_count - if rugged_calls > 0 - payload[:rugged_calls] = rugged_calls - payload[:rugged_duration_s] = Gitlab::RuggedInstrumentation.query_time - end + return if rugged_calls == 0 + + payload[:rugged_calls] = rugged_calls + payload[:rugged_duration_s] = Gitlab::RuggedInstrumentation.query_time + end + + def instrument_redis(payload) + payload.merge! ::Gitlab::Instrumentation::Redis.payload + end + + def instrument_elasticsearch(payload) + # Elasticsearch integration is only available in EE but instrumentation + # only depends on the Gem which is also available in FOSS. + elasticsearch_calls = Gitlab::Instrumentation::ElasticsearchTransport.get_request_count - redis_calls = Gitlab::Instrumentation::Redis.get_request_count + return if elasticsearch_calls == 0 - if redis_calls > 0 - payload[:redis_calls] = redis_calls - payload[:redis_duration_s] = Gitlab::Instrumentation::Redis.query_time - end + payload[:elasticsearch_calls] = elasticsearch_calls + payload[:elasticsearch_duration_s] = Gitlab::Instrumentation::ElasticsearchTransport.query_time end # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the diff --git a/lib/gitlab/issuable_metadata.rb b/lib/gitlab/issuable_metadata.rb index 6f760751b0f..e946fc00c4d 100644 --- a/lib/gitlab/issuable_metadata.rb +++ b/lib/gitlab/issuable_metadata.rb @@ -1,8 +1,52 @@ # frozen_string_literal: true module Gitlab - module IssuableMetadata - def issuable_meta_data(issuable_collection, collection_type, user = nil) + class IssuableMetadata + include Gitlab::Utils::StrongMemoize + + # data structure to store issuable meta data like + # upvotes, downvotes, notes and closing merge requests counts for issues and merge requests + # this avoiding n+1 queries when loading issuable collections on frontend + IssuableMeta = Struct.new(:upvotes, :downvotes, :user_notes_count, :mrs_count) do + def merge_requests_count(user = nil) + mrs_count + end + end + + attr_reader :current_user, :issuable_collection + + def initialize(current_user, issuable_collection) + @current_user = current_user + @issuable_collection = issuable_collection + + validate_collection! + end + + def data + return {} if issuable_ids.empty? + + issuable_ids.each_with_object({}) do |id, issuable_meta| + issuable_meta[id] = metadata_for_issuable(id) + end + end + + private + + def metadata_for_issuable(id) + downvotes = group_issuable_votes_count.find { |votes| votes.awardable_id == id && votes.downvote? } + upvotes = group_issuable_votes_count.find { |votes| votes.awardable_id == id && votes.upvote? } + notes = grouped_issuable_notes_count.find { |notes| notes.noteable_id == id } + merge_requests = grouped_issuable_merge_requests_count.find { |mr| mr.first == id } + + IssuableMeta.new( + upvotes.try(:count).to_i, + downvotes.try(:count).to_i, + notes.try(:count).to_i, + merge_requests.try(:last).to_i + ) + end + + def validate_collection! # ActiveRecord uses Object#extend for null relations. if !(issuable_collection.singleton_class < ActiveRecord::NullRelation) && issuable_collection.respond_to?(:limit_value) && @@ -10,36 +54,43 @@ module Gitlab raise 'Collection must have a limit applied for preloading meta-data' end + end - # map has to be used here since using pluck or select will - # throw an error when ordering issuables by priority which inserts - # a new order into the collection. - # We cannot use reorder to not mess up the paginated collection. - issuable_ids = issuable_collection.map(&:id) + def issuable_ids + strong_memoize(:issuable_ids) do + # map has to be used here since using pluck or select will + # throw an error when ordering issuables by priority which inserts + # a new order into the collection. + # We cannot use reorder to not mess up the paginated collection. + issuable_collection.map(&:id) + end + end - return {} if issuable_ids.empty? + def collection_type + # Supports relations or paginated arrays + issuable_collection.try(:model)&.name || + issuable_collection.first&.model_name.to_s + end - issuable_notes_count = ::Note.count_for_collection(issuable_ids, collection_type) - issuable_votes_count = ::AwardEmoji.votes_for_collection(issuable_ids, collection_type) - issuable_merge_requests_count = + def group_issuable_votes_count + strong_memoize(:group_issuable_votes_count) do + AwardEmoji.votes_for_collection(issuable_ids, collection_type) + end + end + + def grouped_issuable_notes_count + strong_memoize(:grouped_issuable_notes_count) do + ::Note.count_for_collection(issuable_ids, collection_type) + end + end + + def grouped_issuable_merge_requests_count + strong_memoize(:grouped_issuable_merge_requests_count) do if collection_type == 'Issue' - ::MergeRequestsClosingIssues.count_for_collection(issuable_ids, user) + ::MergeRequestsClosingIssues.count_for_collection(issuable_ids, current_user) else [] end - - issuable_ids.each_with_object({}) do |id, issuable_meta| - downvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.downvote? } - upvotes = issuable_votes_count.find { |votes| votes.awardable_id == id && votes.upvote? } - notes = issuable_notes_count.find { |notes| notes.noteable_id == id } - merge_requests = issuable_merge_requests_count.find { |mr| mr.first == id } - - issuable_meta[id] = ::Issuable::IssuableMeta.new( - upvotes.try(:count).to_i, - downvotes.try(:count).to_i, - notes.try(:count).to_i, - merge_requests.try(:last).to_i - ) end end end diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index b973244a531..c09d8170d17 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -12,12 +12,7 @@ module Gitlab def request(*args) result = make_request(*args) - unless result.response.is_a?(Net::HTTPSuccess) - Gitlab::ErrorTracking.track_and_raise_exception( - JIRA::HTTPError.new(result.response), - response: result.body - ) - end + raise JIRA::HTTPError.new(result.response) unless result.response.is_a?(Net::HTTPSuccess) result end diff --git a/lib/gitlab/jira_import.rb b/lib/gitlab/jira_import.rb index 3f56094956a..75d6fdc07b6 100644 --- a/lib/gitlab/jira_import.rb +++ b/lib/gitlab/jira_import.rb @@ -7,11 +7,30 @@ module Gitlab FAILED_ISSUES_COUNTER_KEY = 'jira-import/failed/%{project_id}/%{collection_type}' NEXT_ITEMS_START_AT_KEY = 'jira-import/paginator/%{project_id}/%{collection_type}' JIRA_IMPORT_LABEL = 'jira-import/import-label/%{project_id}' - ITEMS_MAPPER_CACHE_KEY = 'jira-import/items-mapper/%{project_id}/%{collection_type}/%{jira_isssue_id}' + ITEMS_MAPPER_CACHE_KEY = 'jira-import/items-mapper/%{project_id}/%{collection_type}/%{jira_item_id}' + USERS_MAPPER_KEY_PREFIX = 'jira-import/items-mapper/%{project_id}/users/' ALREADY_IMPORTED_ITEMS_CACHE_KEY = 'jira-importer/already-imported/%{project}/%{collection_type}' - def self.jira_issue_cache_key(project_id, jira_issue_id) - ITEMS_MAPPER_CACHE_KEY % { project_id: project_id, collection_type: :issues, jira_isssue_id: jira_issue_id } + def self.validate_project_settings!(project, user: nil, configuration_check: true) + if user + raise Projects::ImportService::Error, _('Cannot import because issues are not available in this project.') unless project.feature_available?(:issues, user) + raise Projects::ImportService::Error, _('You do not have permissions to run the import.') unless user.can?(:admin_project, project) + end + + return unless configuration_check + + jira_service = project.jira_service + + raise Projects::ImportService::Error, _('Jira integration not configured.') unless jira_service&.active? + raise Projects::ImportService::Error, _('Unable to connect to the Jira instance. Please check your Jira integration configuration.') unless jira_service&.valid_connection? + end + + def self.jira_item_cache_key(project_id, jira_item_id, collection_type) + ITEMS_MAPPER_CACHE_KEY % { project_id: project_id, collection_type: collection_type, jira_item_id: jira_item_id } + end + + def self.jira_user_key_prefix(project_id) + USERS_MAPPER_KEY_PREFIX % { project_id: project_id } end def self.already_imported_cache_key(collection_type, project_id) @@ -48,7 +67,7 @@ module Gitlab end def self.cache_issue_mapping(issue_id, jira_issue_id, project_id) - cache_key = JiraImport.jira_issue_cache_key(project_id, jira_issue_id) + cache_key = JiraImport.jira_item_cache_key(project_id, jira_issue_id, :issues) cache_class.write(cache_key, issue_id) end @@ -67,6 +86,19 @@ module Gitlab cache_class.expire(self.already_imported_cache_key(:issues, project_id), JIRA_IMPORT_CACHE_TIMEOUT) end + # Caches the mapping of jira_account_id -> gitlab user id + # project_id - id of a project + # mapping - hash in format of jira_account_id -> gitlab user id + def self.cache_users_mapping(project_id, mapping) + cache_class.write_multiple(mapping, key_prefix: jira_user_key_prefix(project_id)) + end + + def self.get_user_mapping(project_id, jira_account_id) + cache_key = JiraImport.jira_item_cache_key(project_id, jira_account_id, :users) + + cache_class.read(cache_key)&.to_i + end + def self.cache_class Gitlab::Cache::Import::Caching end diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb index 306736df30f..688254bf91f 100644 --- a/lib/gitlab/jira_import/base_importer.rb +++ b/lib/gitlab/jira_import/base_importer.rb @@ -6,7 +6,7 @@ module Gitlab attr_reader :project, :client, :formatter, :jira_project_key, :running_import def initialize(project) - project.validate_jira_import_settings! + Gitlab::JiraImport.validate_project_settings!(project) @running_import = project.latest_jira_import @jira_project_key = running_import&.jira_project_key diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 8c18e58d9df..26fa01755d1 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -57,17 +57,27 @@ module Gitlab # For such cases we exit early if issue was already imported. next if already_imported?(jira_issue.id) - issue_attrs = IssueSerializer.new(project, jira_issue, running_import.user_id, { iid: next_iid }).execute - Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key) - - job_waiter.jobs_remaining += 1 - next_iid += 1 - - # Mark the issue as imported immediately so we don't end up - # importing it multiple times within same import. - # These ids are cleaned-up when import finishes. - # see Gitlab::JiraImport::Stage::FinishImportWorker - mark_as_imported(jira_issue.id) + begin + issue_attrs = IssueSerializer.new(project, jira_issue, running_import.user_id, { iid: next_iid }).execute + + Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key) + + job_waiter.jobs_remaining += 1 + next_iid += 1 + + # Mark the issue as imported immediately so we don't end up + # importing it multiple times within same import. + # These ids are cleaned-up when import finishes. + # see Gitlab::JiraImport::Stage::FinishImportWorker + mark_as_imported(jira_issue.id) + rescue => ex + # handle exceptionn here and skip the failed to import issue, instead of + # failing to import the entire batch of issues + + # track the failed to import issue. + Gitlab::ErrorTracking.track_exception(ex, project_id: project.id) + JiraImport.increment_issue_failures(project.id) + end end job_waiter diff --git a/lib/gitlab/kubernetes/helm.rb b/lib/gitlab/kubernetes/helm.rb index 00ab7109267..9507f7bc117 100644 --- a/lib/gitlab/kubernetes/helm.rb +++ b/lib/gitlab/kubernetes/helm.rb @@ -10,12 +10,6 @@ module Gitlab SERVICE_ACCOUNT = 'tiller' CLUSTER_ROLE_BINDING = 'tiller-admin' CLUSTER_ROLE = 'cluster-admin' - - MANAGED_APPS_LOCAL_TILLER_FEATURE_FLAG = :managed_apps_local_tiller - - def self.local_tiller_enabled? - Feature.enabled?(MANAGED_APPS_LOCAL_TILLER_FEATURE_FLAG) - end end end end diff --git a/lib/gitlab/kubernetes/helm/base_command.rb b/lib/gitlab/kubernetes/helm/base_command.rb index 31cd21f17e0..f27ad05599e 100644 --- a/lib/gitlab/kubernetes/helm/base_command.rb +++ b/lib/gitlab/kubernetes/helm/base_command.rb @@ -3,7 +3,24 @@ module Gitlab module Kubernetes module Helm - module BaseCommand + class BaseCommand + attr_reader :name, :files + + def initialize(rbac:, name:, files:, local_tiller_enabled:) + @rbac = rbac + @name = name + @files = files + @local_tiller_enabled = local_tiller_enabled + end + + def rbac? + @rbac + end + + def local_tiller_enabled? + @local_tiller_enabled + end + def pod_resource pod_service_account_name = rbac? ? service_account_name : nil @@ -46,18 +63,6 @@ module Gitlab files.keys end - def name - raise "Not implemented" - end - - def rbac? - raise "Not implemented" - end - - def files - raise "Not implemented" - end - private def files_dir diff --git a/lib/gitlab/kubernetes/helm/client_command.rb b/lib/gitlab/kubernetes/helm/client_command.rb index e7ade7e4d39..24458e1b4b3 100644 --- a/lib/gitlab/kubernetes/helm/client_command.rb +++ b/lib/gitlab/kubernetes/helm/client_command.rb @@ -57,10 +57,6 @@ module Gitlab '--tls-key', "#{files_dir}/key.pem" ] end - - def local_tiller_enabled? - ::Gitlab::Kubernetes::Helm.local_tiller_enabled? - end end end end diff --git a/lib/gitlab/kubernetes/helm/delete_command.rb b/lib/gitlab/kubernetes/helm/delete_command.rb index 771444ee9ee..3bb41d09994 100644 --- a/lib/gitlab/kubernetes/helm/delete_command.rb +++ b/lib/gitlab/kubernetes/helm/delete_command.rb @@ -3,17 +3,13 @@ module Gitlab module Kubernetes module Helm - class DeleteCommand - include BaseCommand + class DeleteCommand < BaseCommand include ClientCommand attr_reader :predelete, :postdelete - attr_accessor :name, :files - def initialize(name:, rbac:, files:, predelete: nil, postdelete: nil) - @name = name - @files = files - @rbac = rbac + def initialize(predelete: nil, postdelete: nil, **args) + super(**args) @predelete = predelete @postdelete = postdelete end @@ -32,10 +28,6 @@ module Gitlab "uninstall-#{name}" end - def rbac? - @rbac - end - def delete_command command = ['helm', 'delete', '--purge', name] + tls_flags_if_remote_tiller diff --git a/lib/gitlab/kubernetes/helm/init_command.rb b/lib/gitlab/kubernetes/helm/init_command.rb index 058f38f2c9c..e4844e255c5 100644 --- a/lib/gitlab/kubernetes/helm/init_command.rb +++ b/lib/gitlab/kubernetes/helm/init_command.rb @@ -3,27 +3,13 @@ module Gitlab module Kubernetes module Helm - class InitCommand - include BaseCommand - - attr_reader :name, :files - - def initialize(name:, files:, rbac:) - @name = name - @files = files - @rbac = rbac - end - + class InitCommand < BaseCommand def generate_script super + [ init_helm_command ].join("\n") end - def rbac? - @rbac - end - private def init_helm_command diff --git a/lib/gitlab/kubernetes/helm/install_command.rb b/lib/gitlab/kubernetes/helm/install_command.rb index 3784aecccb5..cf6d993cad4 100644 --- a/lib/gitlab/kubernetes/helm/install_command.rb +++ b/lib/gitlab/kubernetes/helm/install_command.rb @@ -3,19 +3,16 @@ module Gitlab module Kubernetes module Helm - class InstallCommand - include BaseCommand + class InstallCommand < BaseCommand include ClientCommand - attr_reader :name, :files, :chart, :repository, :preinstall, :postinstall + attr_reader :chart, :repository, :preinstall, :postinstall attr_accessor :version - def initialize(name:, chart:, files:, rbac:, version: nil, repository: nil, preinstall: nil, postinstall: nil) - @name = name + def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args) + super(**args) @chart = chart @version = version - @rbac = rbac - @files = files @repository = repository @preinstall = preinstall @postinstall = postinstall @@ -33,10 +30,6 @@ module Gitlab ].compact.join("\n") end - def rbac? - @rbac - end - private # Uses `helm upgrade --install` which means we can use this for both diff --git a/lib/gitlab/kubernetes/helm/patch_command.rb b/lib/gitlab/kubernetes/helm/patch_command.rb index ed7a5c2b2d6..1a5fab116bd 100644 --- a/lib/gitlab/kubernetes/helm/patch_command.rb +++ b/lib/gitlab/kubernetes/helm/patch_command.rb @@ -5,23 +5,21 @@ module Gitlab module Kubernetes module Helm - class PatchCommand - include BaseCommand + class PatchCommand < BaseCommand include ClientCommand - attr_reader :name, :files, :chart, :repository + attr_reader :chart, :repository attr_accessor :version - def initialize(name:, chart:, files:, rbac:, version:, repository: nil) + def initialize(chart:, version:, repository: nil, **args) + super(**args) + # version is mandatory to prevent chart mismatches # we do not want our values interpreted in the context of the wrong version raise ArgumentError, 'version is required' if version.blank? - @name = name @chart = chart @version = version - @rbac = rbac - @files = files @repository = repository end @@ -35,10 +33,6 @@ module Gitlab ].compact.join("\n") end - def rbac? - @rbac - end - private def upgrade_command diff --git a/lib/gitlab/kubernetes/helm/reset_command.rb b/lib/gitlab/kubernetes/helm/reset_command.rb index 13176360227..f1f7938039c 100644 --- a/lib/gitlab/kubernetes/helm/reset_command.rb +++ b/lib/gitlab/kubernetes/helm/reset_command.rb @@ -3,18 +3,9 @@ module Gitlab module Kubernetes module Helm - class ResetCommand - include BaseCommand + class ResetCommand < BaseCommand include ClientCommand - attr_reader :name, :files - - def initialize(name:, rbac:, files:) - @name = name - @files = files - @rbac = rbac - end - def generate_script super + [ reset_helm_command, @@ -23,10 +14,6 @@ module Gitlab ].join("\n") end - def rbac? - @rbac - end - def pod_name "uninstall-#{name}" end diff --git a/lib/gitlab/kubernetes/network_policy.rb b/lib/gitlab/kubernetes/network_policy.rb index ea25d81cbd2..dc13a614551 100644 --- a/lib/gitlab/kubernetes/network_policy.rb +++ b/lib/gitlab/kubernetes/network_policy.rb @@ -3,9 +3,12 @@ module Gitlab module Kubernetes class NetworkPolicy - def initialize(name:, namespace:, pod_selector:, ingress:, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil) + DISABLED_BY_LABEL = :'network-policy.gitlab.com/disabled_by' + + def initialize(name:, namespace:, pod_selector:, ingress:, labels: nil, creation_timestamp: nil, policy_types: ["Ingress"], egress: nil) @name = name @namespace = namespace + @labels = labels @creation_timestamp = creation_timestamp @pod_selector = pod_selector @policy_types = policy_types @@ -24,6 +27,7 @@ module Gitlab self.new( name: metadata[:name], namespace: metadata[:namespace], + labels: metadata[:labels], pod_selector: spec[:podSelector], policy_types: spec[:policyTypes], ingress: spec[:ingress], @@ -42,6 +46,7 @@ module Gitlab self.new( name: metadata[:name], namespace: metadata[:namespace], + labels: metadata[:labels]&.to_h, creation_timestamp: metadata[:creationTimestamp], pod_selector: spec[:podSelector], policy_types: spec[:policyTypes], @@ -62,16 +67,48 @@ module Gitlab name: name, namespace: namespace, creation_timestamp: creation_timestamp, - manifest: manifest + manifest: manifest, + is_autodevops: autodevops?, + is_enabled: enabled? } end + def autodevops? + return false unless labels + + !labels[:chart].nil? && labels[:chart].start_with?('auto-deploy-app-') + end + + # podSelector selects pods that should be targeted by this + # policy. We can narrow selection by requiring this policy to + # match our custom labels. Since DISABLED_BY label will not be + # on any pod a policy will be effectively disabled. + def enabled? + return true unless pod_selector&.key?(:matchLabels) + + !pod_selector[:matchLabels]&.key?(DISABLED_BY_LABEL) + end + + def enable + return if enabled? + + pod_selector[:matchLabels].delete(DISABLED_BY_LABEL) + end + + def disable + @pod_selector ||= {} + pod_selector[:matchLabels] ||= {} + pod_selector[:matchLabels].merge!(DISABLED_BY_LABEL => 'gitlab') + end + private - attr_reader :name, :namespace, :creation_timestamp, :pod_selector, :policy_types, :ingress, :egress + attr_reader :name, :namespace, :labels, :creation_timestamp, :pod_selector, :policy_types, :ingress, :egress def metadata - { name: name, namespace: namespace } + meta = { name: name, namespace: namespace } + meta[:labels] = labels if labels + meta end def spec diff --git a/lib/gitlab/lfs_token.rb b/lib/gitlab/lfs_token.rb index e90f3f05a33..c7f2adb27d1 100644 --- a/lib/gitlab/lfs_token.rb +++ b/lib/gitlab/lfs_token.rb @@ -14,7 +14,7 @@ module Gitlab include LfsTokenHelper - DEFAULT_EXPIRE_TIME = 1800 + DEFAULT_EXPIRE_TIME = 7200 # Default value 2 hours attr_accessor :actor diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index 55c46c365f6..17a36c292c0 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -12,24 +12,24 @@ module Gitlab params = event .payload[:params] .each_with_object([]) { |(k, v), array| array << { key: k, value: v } unless IGNORE_PARAMS.include?(k) } - payload = { time: Time.now.utc.iso8601(3), params: Gitlab::Utils::LogLimitedArray.log_limited_array(params, sentinel: LIMITED_ARRAY_SENTINEL), remote_ip: event.payload[:remote_ip], user_id: event.payload[:user_id], username: event.payload[:username], - ua: event.payload[:ua], - queue_duration_s: event.payload[:queue_duration_s] + ua: event.payload[:ua] } + add_db_counters!(payload) payload.merge!(event.payload[:metadata]) if event.payload[:metadata] ::Gitlab::InstrumentationHelper.add_instrumentation_data(payload) + payload[:queue_duration_s] = event.payload[:queue_duration_s] if event.payload[:queue_duration_s] payload[:response] = event.payload[:response] if event.payload[:response] payload[:etag_route] = event.payload[:etag_route] if event.payload[:etag_route] - payload[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id + payload[Labkit::Correlation::CorrelationId::LOG_KEY] = event.payload[Labkit::Correlation::CorrelationId::LOG_KEY] || Labkit::Correlation::CorrelationId.current_id if cpu_s = Gitlab::Metrics::System.thread_cpu_duration(::Gitlab::RequestContext.instance.start_thread_cpu_time) payload[:cpu_s] = cpu_s.round(2) @@ -46,6 +46,16 @@ module Gitlab payload end + + def self.add_db_counters!(payload) + current_transaction = Gitlab::Metrics::Transaction.current + if current_transaction + payload[:db_count] = current_transaction.get(:db_count, :counter).to_i + payload[:db_write_count] = current_transaction.get(:db_write_count, :counter).to_i + payload[:db_cached_count] = current_transaction.get(:db_cached_count, :counter).to_i + end + end + private_class_method :add_db_counters! end end end diff --git a/lib/gitlab/looping_batcher.rb b/lib/gitlab/looping_batcher.rb deleted file mode 100644 index adf0aeda506..00000000000 --- a/lib/gitlab/looping_batcher.rb +++ /dev/null @@ -1,99 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - # Returns an ID range within a table so it can be iterated over. Repeats from - # the beginning after it reaches the end. - # - # Used by Geo in particular to iterate over a replicable and its registry - # table. - # - # Tracks a cursor for each table, by "key". If the table is smaller than - # batch_size, then a range for the whole table is returned on every call. - class LoopingBatcher - # @param [Class] model_class the class of the table to iterate on - # @param [String] key to identify the cursor. Note, cursor is already unique - # per table. - # @param [Integer] batch_size to limit the number of records in a batch - def initialize(model_class, key:, batch_size: 1000) - @model_class = model_class - @key = key - @batch_size = batch_size - end - - # @return [Range] a range of IDs. `nil` if 0 records at or after the cursor. - def next_range! - return unless @model_class.any? - - batch_first_id = cursor_id - - batch_last_id = get_batch_last_id(batch_first_id) - return unless batch_last_id - - batch_first_id..batch_last_id - end - - private - - # @private - # - # Get the last ID of the batch. Increment the cursor or reset it if at end. - # - # @param [Integer] batch_first_id the first ID of the batch - # @return [Integer] batch_last_id the last ID of the batch (not the table) - def get_batch_last_id(batch_first_id) - batch_last_id, more_rows = run_query(@model_class.table_name, @model_class.primary_key, batch_first_id, @batch_size) - - if more_rows - increment_batch(batch_last_id) - else - reset if batch_first_id > 1 - end - - batch_last_id - end - - def run_query(table, primary_key, batch_first_id, batch_size) - sql = <<~SQL - SELECT MAX(batch.id) AS batch_last_id, - EXISTS ( - SELECT #{primary_key} - FROM #{table} - WHERE #{primary_key} > MAX(batch.id) - ) AS more_rows - FROM ( - SELECT #{primary_key} - FROM #{table} - WHERE #{primary_key} >= #{batch_first_id} - ORDER BY #{primary_key} - LIMIT #{batch_size}) AS batch; - SQL - - result = ActiveRecord::Base.connection.exec_query(sql).first - - [result["batch_last_id"], result["more_rows"]] - end - - def reset - set_cursor_id(1) - end - - def increment_batch(batch_last_id) - set_cursor_id(batch_last_id + 1) - end - - # @private - # - # @return [Integer] the cursor ID, or 1 if it is not set - def cursor_id - Rails.cache.fetch("#{cache_key}:cursor_id") || 1 - end - - def set_cursor_id(id) - Rails.cache.write("#{cache_key}:cursor_id", id) - end - - def cache_key - @cache_key ||= "#{self.class.name.parameterize}:#{@model_class.name.parameterize}:#{@key}:cursor_id" - end - end -end diff --git a/lib/gitlab/metrics/dashboard/stages/url_validator.rb b/lib/gitlab/metrics/dashboard/stages/url_validator.rb new file mode 100644 index 00000000000..ff36f7b605e --- /dev/null +++ b/lib/gitlab/metrics/dashboard/stages/url_validator.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Dashboard + module Stages + class UrlValidator < BaseStage + def transform! + dashboard[:links]&.each do |link| + Gitlab::UrlBlocker.validate!(link[:url]) + rescue Gitlab::UrlBlocker::BlockedUrlError + link[:url] = '' + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/elasticsearch_rack_middleware.rb b/lib/gitlab/metrics/elasticsearch_rack_middleware.rb new file mode 100644 index 00000000000..6830eed68d5 --- /dev/null +++ b/lib/gitlab/metrics/elasticsearch_rack_middleware.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + # Rack middleware for tracking Elasticsearch metrics from Grape and Web requests. + class ElasticsearchRackMiddleware + HISTOGRAM_BUCKETS = [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 60].freeze + + def initialize(app) + @app = app + + @requests_total_counter = Gitlab::Metrics.counter(:http_elasticsearch_requests_total, + 'Amount of calls to Elasticsearch servers during web requests', + Gitlab::Metrics::Transaction::BASE_LABELS) + @requests_duration_histogram = Gitlab::Metrics.histogram(:http_elasticsearch_requests_duration_seconds, + 'Query time for Elasticsearch servers during web requests', + Gitlab::Metrics::Transaction::BASE_LABELS, + HISTOGRAM_BUCKETS) + end + + def call(env) + transaction = Gitlab::Metrics.current_transaction + + @app.call(env) + ensure + record_metrics(transaction) + end + + private + + def record_metrics(transaction) + labels = transaction.labels + query_time = ::Gitlab::Instrumentation::ElasticsearchTransport.query_time + request_count = ::Gitlab::Instrumentation::ElasticsearchTransport.get_request_count + + @requests_total_counter.increment(labels, request_count) + @requests_duration_histogram.observe(labels, query_time) + end + end + end +end diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index cee601ff14c..5955987541c 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -52,7 +52,7 @@ module Gitlab end def disabled_by_feature(options) - options.with_feature && !::Feature.get(options.with_feature).enabled? + options.with_feature && !::Feature.enabled?(options.with_feature) end def build_metric!(type, name, options) diff --git a/lib/gitlab/metrics/redis_rack_middleware.rb b/lib/gitlab/metrics/redis_rack_middleware.rb new file mode 100644 index 00000000000..f0f99c5f45d --- /dev/null +++ b/lib/gitlab/metrics/redis_rack_middleware.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + # Rack middleware for tracking Redis metrics from Grape and Web requests. + class RedisRackMiddleware + def initialize(app) + @app = app + + @requests_total_counter = Gitlab::Metrics.counter(:http_redis_requests_total, + 'Amount of calls to Redis servers during web requests', + Gitlab::Metrics::Transaction::BASE_LABELS) + @requests_duration_histogram = Gitlab::Metrics.histogram(:http_redis_requests_duration_seconds, + 'Query time for Redis servers during web requests', + Gitlab::Metrics::Transaction::BASE_LABELS, + Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS) + end + + def call(env) + transaction = Gitlab::Metrics.current_transaction + + @app.call(env) + ensure + record_metrics(transaction) + end + + private + + def record_metrics(transaction) + labels = transaction.labels + query_time = Gitlab::Instrumentation::Redis.query_time + request_count = Gitlab::Instrumentation::Redis.get_request_count + + @requests_total_counter.increment(labels, request_count) + @requests_duration_histogram.observe(labels, query_time) + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 90051f85f31..ff3e7be567f 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -6,8 +6,10 @@ module Gitlab module Metrics module Samplers class BaseSampler < Daemon + attr_reader :interval + # interval - The sampling interval in seconds. - def initialize(interval) + def initialize(interval = self.class::SAMPLING_INTERVAL_SECONDS) interval_half = interval.to_f / 2 @interval = interval diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index 98dd517ee3b..b5343d5e66a 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -4,6 +4,8 @@ module Gitlab module Metrics module Samplers class PumaSampler < BaseSampler + SAMPLING_INTERVAL_SECONDS = 5 + def metrics @metrics ||= init_metrics end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index df59c06911b..dac9fbd1247 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -6,9 +6,10 @@ module Gitlab module Metrics module Samplers class RubySampler < BaseSampler + SAMPLING_INTERVAL_SECONDS = 60 GC_REPORT_BUCKETS = [0.005, 0.01, 0.02, 0.04, 0.07, 0.1, 0.5].freeze - def initialize(interval) + def initialize(*) GC::Profiler.clear metrics[:process_start_time_seconds].set(labels, Time.now.to_i) diff --git a/lib/gitlab/metrics/sidekiq_middleware.rb b/lib/gitlab/metrics/sidekiq_middleware.rb index 8dfb61046c4..de8e1ca3256 100644 --- a/lib/gitlab/metrics/sidekiq_middleware.rb +++ b/lib/gitlab/metrics/sidekiq_middleware.rb @@ -6,19 +6,30 @@ module Gitlab # # This middleware is intended to be used as a server-side middleware. class SidekiqMiddleware - def call(worker, message, queue) + def call(worker, payload, queue) trans = BackgroundTransaction.new(worker.class) begin # Old gitlad-shell messages don't provide enqueued_at/created_at attributes - trans.set(:sidekiq_queue_duration, Time.now.to_f - (message['enqueued_at'] || message['created_at'] || 0)) + enqueued_at = payload['enqueued_at'] || payload['created_at'] || 0 + trans.set(:sidekiq_queue_duration, Time.current.to_f - enqueued_at) trans.run { yield } rescue Exception => error # rubocop: disable Lint/RescueException trans.add_event(:sidekiq_exception) raise error + ensure + add_info_to_payload(payload, trans) end end + + private + + def add_info_to_payload(payload, trans) + payload[:db_count] = trans.get(:db_count, :counter).to_i + payload[:db_write_count] = trans.get(:db_write_count, :counter).to_i + payload[:db_cached_count] = trans.get(:db_cached_count, :counter).to_i + end end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index a02dd850582..1628eeb5a95 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -9,6 +9,7 @@ module Gitlab attach_to :active_record IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze + DB_COUNTERS = %i{db_count db_write_count db_cached_count}.freeze def sql(event) return unless current_transaction @@ -19,8 +20,7 @@ module Gitlab self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0) - current_transaction.increment(:sql_duration, event.duration, false) - current_transaction.increment(:sql_count, 1, false) + increment_db_counters(payload) end private @@ -31,6 +31,20 @@ module Gitlab buckets [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0] end + def select_sql_command?(payload) + payload[:sql].match(/\A((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i) + end + + def increment_db_counters(payload) + current_transaction.increment(:db_count, 1) + + if payload.fetch(:cached, payload[:name] == 'CACHE') + current_transaction.increment(:db_cached_count, 1) + end + + current_transaction.increment(:db_write_count, 1) unless select_sql_command?(payload) + end + def current_transaction Transaction.current end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index b126efd2dd5..822f5243e9d 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -16,7 +16,7 @@ module Gitlab # The series to store events (e.g. Git pushes) in. EVENT_SERIES = 'events' - attr_reader :tags, :method + attr_reader :method def self.current Thread.current[THREAD_KEY] @@ -28,8 +28,6 @@ module Gitlab @started_at = nil @finished_at = nil - @tags = {} - @memory_before = 0 @memory_after = 0 end @@ -94,6 +92,12 @@ module Gitlab self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus end + def get(name, type, tags = {}) + metric = self.class.transaction_metric(name, type) + + metric.get(filter_tags(tags).merge(labels)) + end + def labels BASE_LABELS end diff --git a/lib/gitlab/middleware/handle_ip_spoof_attack_error.rb b/lib/gitlab/middleware/handle_ip_spoof_attack_error.rb new file mode 100644 index 00000000000..2fc08db9b4d --- /dev/null +++ b/lib/gitlab/middleware/handle_ip_spoof_attack_error.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Middleware + # ActionDispatch::RemoteIp tries to set the `request.ip` for controllers by + # looking at the request IP and headers. It needs to see through any reverse + # proxies to get the right answer, but there are some security issues with + # that. + # + # Proxies can specify `Client-Ip` or `X-Forwarded-For`, and the security of + # that is determined at the edge. If both headers are present, it's likely + # that the edge is securing one, but ignoring the other. Rails blocks this, + # which is correct, because we don't know which header is the safe one - but + # we want the block to be a 400, rather than 500, error. + # + # This middleware needs to go before ActionDispatch::RemoteIp in the chain. + class HandleIpSpoofAttackError + attr_reader :app + + def initialize(app) + @app = app + end + + def call(env) + app.call(env) + rescue ActionDispatch::RemoteIp::IpSpoofAttackError => err + Gitlab::ErrorTracking.track_exception(err) + + [400, { 'Content-Type' => 'text/plain' }, ['Bad Request']] + end + end + end +end diff --git a/lib/gitlab/monitor/demo_projects.rb b/lib/gitlab/monitor/demo_projects.rb new file mode 100644 index 00000000000..c617f895e4c --- /dev/null +++ b/lib/gitlab/monitor/demo_projects.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Monitor + # See Demo Project documentation + # https://about.gitlab.com/handbook/engineering/development/ops/monitor/#demo-environments + module DemoProjects + # [https://gitlab.com/gitlab-org/monitor/tanuki-inc, https://gitlab.com/gitlab-org/monitor/monitor-sandbox] + DOT_COM_IDS = [14986497, 12507547].freeze + # [https://staging.gitlab.com/gitlab-org/monitor/monitor-sandbox] + STAGING_IDS = [4422333].freeze + + def self.primary_keys + # .com? returns true for staging + if ::Gitlab.com? && !::Gitlab.staging? + DOT_COM_IDS + elsif ::Gitlab.staging? + STAGING_IDS + elsif ::Gitlab.dev_or_test_env? + Project.limit(100).pluck(:id) # rubocop: disable CodeReuse/ActiveRecord + else + [] + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/request_context.rb b/lib/gitlab/pagination/keyset/request_context.rb index 8c8138b3076..070fa844347 100644 --- a/lib/gitlab/pagination/keyset/request_context.rb +++ b/lib/gitlab/pagination/keyset/request_context.rb @@ -24,7 +24,9 @@ module Gitlab end def apply_headers(next_page) - request.header('Links', pagination_links(next_page)) + link = pagination_links(next_page) + request.header('Links', link) + request.header('Link', link) end private diff --git a/lib/gitlab/pagination/offset_pagination.rb b/lib/gitlab/pagination/offset_pagination.rb index 11a5ef4e518..8796dd4d7ec 100644 --- a/lib/gitlab/pagination/offset_pagination.rb +++ b/lib/gitlab/pagination/offset_pagination.rb @@ -19,7 +19,13 @@ module Gitlab private def paginate_with_limit_optimization(relation) - pagination_data = relation.page(params[:page]).per(params[:per_page]) + # do not paginate relation if it is already paginated + pagination_data = if relation.respond_to?(:current_page) && relation.current_page == params[:page] && relation.limit_value == params[:per_page] + relation + else + relation.page(params[:page]).per(params[:per_page]) + end + return pagination_data unless pagination_data.is_a?(ActiveRecord::Relation) return pagination_data unless Feature.enabled?(:api_kaminari_count_with_limit) diff --git a/lib/gitlab/phabricator_import/cache/map.rb b/lib/gitlab/phabricator_import/cache/map.rb index 6a2841b6a8e..7aba3cf26fd 100644 --- a/lib/gitlab/phabricator_import/cache/map.rb +++ b/lib/gitlab/phabricator_import/cache/map.rb @@ -63,7 +63,7 @@ module Gitlab def timeout # Setting the timeout to the same one as we do for clearing stuck jobs # this makes sure all cache is available while the import is running. - StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION + Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION end end end diff --git a/lib/gitlab/phabricator_import/worker_state.rb b/lib/gitlab/phabricator_import/worker_state.rb index 38829e34509..ffa2d3d7a43 100644 --- a/lib/gitlab/phabricator_import/worker_state.rb +++ b/lib/gitlab/phabricator_import/worker_state.rb @@ -40,7 +40,7 @@ module Gitlab def timeout # Make sure we get rid of all the information after a job is marked # as failed/succeeded - StuckImportJobsWorker::IMPORT_JOBS_EXPIRATION + Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION end end end diff --git a/lib/gitlab/process_memory_cache/helper.rb b/lib/gitlab/process_memory_cache/helper.rb new file mode 100644 index 00000000000..ee4b81a9a19 --- /dev/null +++ b/lib/gitlab/process_memory_cache/helper.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + class ProcessMemoryCache + module Helper + def fetch_memory_cache(key, &payload) + cache = cache_backend.read(key) + + if cache && !stale_cache?(key, cache) + cache[:data] + else + store_cache(key, &payload) + end + end + + def invalidate_memory_cache(key) + touch_cache_timestamp(key) + end + + private + + def touch_cache_timestamp(key, time = Time.current.to_f) + shared_backend.write(key, time) + end + + def stale_cache?(key, cache_info) + shared_timestamp = shared_backend.read(key) + return true unless shared_timestamp + + shared_timestamp.to_f > cache_info[:cached_at].to_f + end + + def store_cache(key) + data = yield + time = Time.current.to_f + + cache_backend.write(key, data: data, cached_at: time) + touch_cache_timestamp(key, time) + data + end + + def shared_backend + Rails.cache + end + + def cache_backend + ::Gitlab::ProcessMemoryCache.cache_backend + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index fbdfe166645..e6b25e71eb3 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -11,7 +11,7 @@ module Gitlab @query = query end - def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE) + def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil) case scope when 'notes' notes.page(page).per(per_page) @@ -20,7 +20,7 @@ module Gitlab when 'wiki_blobs' paginated_wiki_blobs(wiki_blobs(limit: limit_up_to_page(page, per_page)), page, per_page) when 'commits' - Kaminari.paginate_array(commits).page(page).per(per_page) + paginated_commits(page, per_page) when 'users' users.page(page).per(per_page) else @@ -37,7 +37,7 @@ module Gitlab when 'wiki_blobs' wiki_blobs_count.to_s when 'commits' - commits_count.to_s + formatted_limited_count(commits_count) else super end @@ -72,7 +72,7 @@ module Gitlab end def commits_count - @commits_count ||= commits.count + @commits_count ||= commits(limit: count_limit).count end def single_commit_result? @@ -86,6 +86,12 @@ module Gitlab private + def paginated_commits(page, per_page) + results = commits(limit: limit_up_to_page(page, per_page)) + + Kaminari.paginate_array(results).page(page).per(per_page) + end + def paginated_blobs(blobs, page, per_page) results = Kaminari.paginate_array(blobs).page(page).per(per_page) @@ -139,21 +145,21 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def commits - @commits ||= find_commits(query) + def commits(limit:) + @commits ||= find_commits(query, limit: limit) end - def find_commits(query) + def find_commits(query, limit:) return [] unless Ability.allowed?(@current_user, :download_code, @project) - commits = find_commits_by_message(query) + commits = find_commits_by_message(query, limit: limit) commit_by_sha = find_commit_by_sha(query) commits |= [commit_by_sha] if commit_by_sha commits end - def find_commits_by_message(query) - project.repository.find_commits_by_message(query) + def find_commits_by_message(query, limit:) + project.repository.find_commits_by_message(query, repository_project_ref, nil, limit) end def find_commit_by_sha(query) diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 38adfc03ea7..fdb3fbc03bc 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -44,13 +44,13 @@ module Gitlab ProjectTemplate.new('iosswift', 'iOS (Swift)', _('A ready-to-go template for use with iOS Swift apps.'), 'https://gitlab.com/gitlab-org/project-templates/iosswift', 'illustrations/logos/swift.svg'), ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/logos/dotnet.svg'), ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps.'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'), - ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro'), + ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development.'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'), ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby.'), 'https://gitlab.com/pages/gatsby'), - ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo'), - ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll'), + ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo.'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'), + ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll.'), 'https://gitlab.com/pages/jekyll', 'illustrations/logos/jekyll.svg'), ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML.'), 'https://gitlab.com/pages/plain-html'), - ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook'), - ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo'), + ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook.'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'), + ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo.'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'), ProjectTemplate.new('sse_middleman', 'Static Site Editor/Middleman', _('Middleman project with Static Site Editor support'), 'https://gitlab.com/gitlab-org/project-templates/static-site-editor-middleman'), ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features.'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), diff --git a/lib/gitlab/prometheus/query_variables.rb b/lib/gitlab/prometheus/query_variables.rb index 4d48c4a3af7..5b688f83545 100644 --- a/lib/gitlab/prometheus/query_variables.rb +++ b/lib/gitlab/prometheus/query_variables.rb @@ -3,8 +3,10 @@ module Gitlab module Prometheus module QueryVariables - def self.call(environment) + # start_time and end_time should be Time objects. + def self.call(environment, start_time: nil, end_time: nil) { + __range: range(start_time, end_time), ci_environment_slug: environment.slug, kube_namespace: environment.deployment_namespace || '', environment_filter: %{container_name!="POD",environment="#{environment.slug}"}, @@ -14,6 +16,16 @@ module Gitlab ci_environment_name: environment.name } end + + private + + def self.range(start_time, end_time) + if start_time && end_time + range_seconds = (end_time - start_time).to_i + "#{range_seconds}s" + end + end + private_class_method :range end end end diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb index b03d8a4d254..213e3ba835d 100644 --- a/lib/gitlab/prometheus_client.rb +++ b/lib/gitlab/prometheus_client.rb @@ -71,6 +71,19 @@ module Gitlab end end + # Queries Prometheus with the given aggregate query and groups the results by mapping + # metric labels to their respective values. + # + # @return [Hash] mapping labels to their aggregate numeric values, or the empty hash if no results were found + def aggregate(aggregate_query, time: Time.now) + response = query(aggregate_query, time: time) + response.to_h do |result| + key = block_given? ? yield(result['metric']) : result['metric'] + _timestamp, value = result['value'] + [key, value.to_i] + end + end + def label_values(name = '__name__') json_api_get("label/#{name}/values") end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 7c06698ffec..98db8ff761e 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -104,6 +104,23 @@ module Gitlab command :target_branch do |branch_name| @updates[:target_branch] = branch_name if project.repository.branch_exists?(branch_name) end + + desc _('Submit a review') + explanation _('Submit the current review.') + types MergeRequest + condition do + quick_action_target.persisted? + end + command :submit_review do + next if params[:review_id] + + result = DraftNotes::PublishService.new(quick_action_target, current_user).execute + @execution_message[:submit_review] = if result[:status] == :success + _('Submitted the current review.') + else + result[:message] + end + end end def merge_orchestration_service diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 6e31c506438..a634f12345a 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -27,6 +27,10 @@ module Gitlab # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent super end + + def instrumentation_class + ::Gitlab::Instrumentation::Redis::Cache + end end end end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 0375e4a221a..42d5167beb3 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -28,6 +28,10 @@ module Gitlab # this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent super end + + def instrumentation_class + ::Gitlab::Instrumentation::Redis::Queues + end end end end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index 35356083f26..8ab53700932 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -30,6 +30,10 @@ module Gitlab # this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent super end + + def instrumentation_class + ::Gitlab::Instrumentation::Redis::SharedState + end end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 06ee81ba172..5584323789b 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -71,6 +71,10 @@ module Gitlab # nil will force use of DEFAULT_REDIS_URL when config file is absent nil end + + def instrumentation_class + raise NotImplementedError + end end def initialize(rails_env = nil) @@ -100,6 +104,8 @@ module Gitlab redis_url = config.delete(:url) redis_uri = URI.parse(redis_url) + config[:instrumentation_class] ||= self.class.instrumentation_class + if redis_uri.scheme == 'unix' # Redis::Store does not handle Unix sockets well, so let's do it for them config[:path] = redis_uri.path diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index d07d6440c6b..01aff48b08b 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -4,7 +4,7 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project - merge_request snippet commit commit_range directly_addressed_user epic).freeze + merge_request snippet commit commit_range directly_addressed_user epic iteration).freeze attr_accessor :project, :current_user, :author # This counter is increased by a number of references filtered out by # banzai reference exctractor. Note that this counter is stateful and diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index c8b04ce2a5c..4caff8ae679 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -2,7 +2,93 @@ module Gitlab module Regex + module Packages + CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt].freeze + CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze + + def conan_file_name_regex + @conan_file_name_regex ||= + %r{\A#{(CONAN_RECIPE_FILES + CONAN_PACKAGE_FILES).join("|")}\z}.freeze + end + + def conan_package_reference_regex + @conan_package_reference_regex ||= %r{\A[A-Za-z0-9]+\z}.freeze + end + + def conan_revision_regex + @conan_revision_regex ||= %r{\A0\z}.freeze + end + + def conan_recipe_component_regex + @conan_recipe_component_regex ||= %r{\A[a-zA-Z0-9_][a-zA-Z0-9_\+\.-]{1,49}\z}.freeze + end + + def composer_package_version_regex + @composer_package_version_regex ||= %r{^v?(\d+(\.(\d+|x))*(-.+)?)}.freeze + end + + def package_name_regex + @package_name_regex ||= %r{\A\@?(([\w\-\.\+]*)\/)*([\w\-\.]+)@?(([\w\-\.\+]*)\/)*([\w\-\.]*)\z}.freeze + end + + def maven_file_name_regex + @maven_file_name_regex ||= %r{\A[A-Za-z0-9\.\_\-\+]+\z}.freeze + end + + def maven_path_regex + @maven_path_regex ||= %r{\A\@?(([\w\-\.]*)/)*([\w\-\.\+]*)\z}.freeze + end + + def maven_app_name_regex + @maven_app_name_regex ||= /\A[\w\-\.]+\z/.freeze + end + + def maven_app_group_regex + maven_app_name_regex + end + + def unbounded_semver_regex + # See the official regex: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string + + # The order of the alternatives in <prerelease> are intentionally + # reordered to be greedy. Without this change, the unbounded regex would + # only partially match "v0.0.0-20201230123456-abcdefabcdef". + @unbounded_semver_regex ||= / + (?<major>0|[1-9]\d*) + \.(?<minor>0|[1-9]\d*) + \.(?<patch>0|[1-9]\d*) + (?:-(?<prerelease>(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0)(?:\.(?:\d*[a-zA-Z-][0-9a-zA-Z-]*|[1-9]\d*|0))*))? + (?:\+(?<build>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))? + /x.freeze + end + + def semver_regex + @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options) + end + + def go_package_regex + # A Go package name looks like a URL but is not; it: + # - Must not have a scheme, such as http:// or https:// + # - Must not have a port number, such as :8080 or :8443 + + @go_package_regex ||= / + \b (?# word boundary) + (?<domain> + [0-9a-z](?:(?:-|[0-9a-z]){0,61}[0-9a-z])? (?# first domain) + (?:\.[0-9a-z](?:(?:-|[0-9a-z]){0,61}[0-9a-z])?)* (?# inner domains) + \.[a-z]{2,} (?# top-level domain) + ) + (?<path>\/(?: + [-\/$_.+!*'(),0-9a-z] (?# plain URL character) + | %[0-9a-f]{2})* (?# URL encoded character) + )? (?# path) + \b (?# word boundary) + /ix.freeze + end + end + extend self + extend Packages def project_name_regex # The character range \p{Alnum} overlaps with \u{00A9}-\u{1f9ff} @@ -163,6 +249,10 @@ module Gitlab def issue @issue ||= /(?<issue>\d+\b)/ end + + def base64_regex + @base64_regex ||= /(?:[A-Za-z0-9+\/]{4})*(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)?/.freeze + end end end diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb index 84885be9bda..cad127922df 100644 --- a/lib/gitlab/routing.rb +++ b/lib/gitlab/routing.rb @@ -4,6 +4,36 @@ module Gitlab module Routing extend ActiveSupport::Concern + class LegacyRedirector + # @params path_type [symbol] type of path to do "-" redirection + # https://gitlab.com/gitlab-org/gitlab/-/issues/16854 + def initialize(path_type) + @path_type = path_type + end + + def call(_params, request) + ensure_valid_uri!(request) + + # Only replace the last occurrence of `path`. + # + # `request.fullpath` includes the querystring + new_path = request.path.sub(%r{/#{@path_type}(/*)(?!.*#{@path_type})}, "/-/#{@path_type}\\1") + new_path = "#{new_path}?#{request.query_string}" if request.query_string.present? + + new_path + end + + private + + def ensure_valid_uri!(request) + URI.parse(request.path) + rescue URI::InvalidURIError => e + # If url is invalid, raise custom error, + # which can be ignored by monitoring tools. + raise ActionController::RoutingError.new(e.message) + end + end + mattr_accessor :_includers self._includers = [] @@ -44,20 +74,10 @@ module Gitlab end def self.redirect_legacy_paths(router, *paths) - build_redirect_path = lambda do |request, _params, path| - # Only replace the last occurrence of `path`. - # - # `request.fullpath` includes the querystring - new_path = request.path.sub(%r{/#{path}(/*)(?!.*#{path})}, "/-/#{path}\\1") - new_path = "#{new_path}?#{request.query_string}" if request.query_string.present? - - new_path - end - paths.each do |path| router.match "/#{path}(/*rest)", via: [:get, :post, :patch, :delete], - to: router.redirect { |params, request| build_redirect_path.call(request, params, path) }, + to: router.redirect(LegacyRedirector.new(path)), as: "legacy_#{path}_redirect" end end diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb index 9a5917ffba9..36a3a491de6 100644 --- a/lib/gitlab/rugged_instrumentation.rb +++ b/lib/gitlab/rugged_instrumentation.rb @@ -3,12 +3,13 @@ module Gitlab module RuggedInstrumentation def self.query_time - query_time = SafeRequestStore[:rugged_query_time] ||= 0 + query_time = SafeRequestStore[:rugged_query_time] || 0 query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION) end - def self.query_time=(duration) - SafeRequestStore[:rugged_query_time] = duration + def self.add_query_time(duration) + SafeRequestStore[:rugged_query_time] ||= 0 + SafeRequestStore[:rugged_query_time] += duration end def self.query_time_ms diff --git a/lib/gitlab/search_context.rb b/lib/gitlab/search_context.rb new file mode 100644 index 00000000000..c3bb0ff26f2 --- /dev/null +++ b/lib/gitlab/search_context.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +module Gitlab + # Holds the contextual data used by navbar search component to + # determine the search scope, whether to search for code, or if + # a search should target snippets. + # + # Use the SearchContext::Builder to create an instance of this class + class SearchContext + attr_accessor :project, :project_metadata, :ref, + :group, :group_metadata, + :snippets, + :scope, :search_url + + def initialize + @ref = nil + @project = nil + @project_metadata = {} + @group = nil + @group_metadata = {} + @snippets = [] + @scope = nil + @search_url = nil + end + + def for_project? + project.present? && project.persisted? + end + + def for_group? + group.present? && group.persisted? + end + + def for_snippets? + snippets.any? + end + + def code_search? + project.present? && scope.nil? + end + + class Builder + def initialize(view_context) + @view_context = view_context + @snippets = [] + end + + def with_snippet(snippet) + @snippets << snippet + + self + end + + def with_project(project) + @project = project + with_group(project&.group) + + self + end + + def with_group(group) + @group = group + + self + end + + def with_ref(ref) + @ref = ref + + self + end + + def build! + SearchContext.new.tap do |context| + context.project = @project + context.group = @group + context.ref = @ref + context.snippets = @snippets.dup + context.scope = search_scope + context.search_url = search_url + context.group_metadata = group_search_metadata(@group) + context.project_metadata = project_search_metadata(@project) + end + end + + private + + attr_accessor :view_context + + def project_search_metadata(project) + return {} unless project + + { + project_path: project.path, + name: project.name, + issues_path: view_context.project_issues_path(project), + mr_path: view_context.project_merge_requests_path(project), + issues_disabled: !project.issues_enabled? + } + end + + def group_search_metadata(group) + return {} unless group + + { + group_path: group.path, + name: group.name, + issues_path: view_context.issues_group_path(group), + mr_path: view_context.merge_requests_group_path(group) + } + end + + def search_url + if @project.present? + view_context.search_path(project_id: @project.id) + elsif @group.present? + view_context.search_path(group_id: @group.id) + else + view_context.search_path + end + end + + def search_scope + if view_context.current_controller?(:issues) + 'issues' + elsif view_context.current_controller?(:merge_requests) + 'merge_requests' + elsif view_context.current_controller?(:wikis) + 'wiki_blobs' + elsif view_context.current_controller?(:commits) + 'commits' + else nil + end + end + end + + module ControllerConcern + extend ActiveSupport::Concern + + included do + helper_method :search_context + end + + # rubocop:disable Gitlab/ModuleWithInstanceVariables + # + # Introspect the current controller's assignments and + # and builds the proper SearchContext object for it. + def search_context + builder = Builder.new(view_context) + + builder.with_snippet(@snippet) if @snippet.present? + @snippets.each(&builder.method(:with_snippet)) if @snippets.present? + builder.with_project(@project) if @project.present? && @project.persisted? + builder.with_group(@group) if @group.present? && @group.persisted? + builder.with_ref(@ref) if @ref.present? + + builder.build! + end + # rubocop:enable Gitlab/ModuleWithInstanceVariables + end + end +end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index c35ee62163a..6239158ef06 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -26,7 +26,7 @@ module Gitlab @default_project_filter = default_project_filter end - def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, without_count: true) + def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, without_count: true, preload_method: nil) collection = case scope when 'projects' projects diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 3f36725cb66..64a30fbe16c 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -79,6 +79,7 @@ module Gitlab config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } config[:bin_dir] = Gitlab.config.gitaly.client_path + config[:gitlab] = { url: Gitlab.config.gitlab.url } TomlRB.dump(config) end @@ -97,7 +98,8 @@ module Gitlab def configuration_toml(gitaly_dir, storage_paths) nodes = [{ storage: 'default', address: "unix:#{gitaly_dir}/gitaly.socket", primary: true, token: 'secret' }] storages = [{ name: 'default', node: nodes }] - config = { socket_path: "#{gitaly_dir}/praefect.socket", virtual_storage: storages } + failover = { enabled: false } + config = { socket_path: "#{gitaly_dir}/praefect.socket", memory_queue_enabled: true, virtual_storage: storages, failover: failover } config[:token] = 'secret' if Rails.env.test? TomlRB.dump(config) diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 4e0d3da1868..633291dcdf3 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -14,8 +14,8 @@ module Gitlab ].compact.freeze DEFAULT_WORKERS = [ - DummyWorker.new('default', weight: 1), - DummyWorker.new('mailers', weight: 2) + DummyWorker.new('default', weight: 1, tags: []), + DummyWorker.new('mailers', weight: 2, tags: []) ].map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze class << self diff --git a/lib/gitlab/sidekiq_config/cli_methods.rb b/lib/gitlab/sidekiq_config/cli_methods.rb index 0d0efe8ffbd..a256632bc12 100644 --- a/lib/gitlab/sidekiq_config/cli_methods.rb +++ b/lib/gitlab/sidekiq_config/cli_methods.rb @@ -28,6 +28,7 @@ module Gitlab has_external_dependencies: lambda { |value| value == 'true' }, name: :to_s, resource_boundary: :to_sym, + tags: :to_sym, urgency: :to_sym }.freeze @@ -117,7 +118,11 @@ module Gitlab raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block - lambda { |queue| values.map(&values_block).include?(queue[lhs.to_sym]) } + lambda do |queue| + comparator = Array(queue[lhs.to_sym]).to_set + + values.map(&values_block).to_set.intersect?(comparator) + end end end end diff --git a/lib/gitlab/sidekiq_config/dummy_worker.rb b/lib/gitlab/sidekiq_config/dummy_worker.rb index bd205c81931..7568840410b 100644 --- a/lib/gitlab/sidekiq_config/dummy_worker.rb +++ b/lib/gitlab/sidekiq_config/dummy_worker.rb @@ -12,7 +12,8 @@ module Gitlab urgency: :get_urgency, resource_boundary: :get_worker_resource_boundary, idempotent: :idempotent?, - weight: :get_weight + weight: :get_weight, + tags: :get_tags }.freeze def initialize(queue, attributes = {}) diff --git a/lib/gitlab/sidekiq_config/worker.rb b/lib/gitlab/sidekiq_config/worker.rb index ec7a82f6459..46fa0aa5be1 100644 --- a/lib/gitlab/sidekiq_config/worker.rb +++ b/lib/gitlab/sidekiq_config/worker.rb @@ -6,7 +6,7 @@ module Gitlab include Comparable attr_reader :klass - delegate :feature_category_not_owned?, :get_feature_category, + delegate :feature_category_not_owned?, :get_feature_category, :get_tags, :get_urgency, :get_weight, :get_worker_resource_boundary, :idempotent?, :queue, :queue_namespace, :worker_has_external_dependencies?, @@ -52,7 +52,8 @@ module Gitlab urgency: get_urgency, resource_boundary: get_worker_resource_boundary, weight: get_weight, - idempotent: idempotent? + idempotent: idempotent?, + tags: get_tags } end diff --git a/lib/gitlab/sidekiq_logging/json_formatter.rb b/lib/gitlab/sidekiq_logging/json_formatter.rb index 64782e1e1d1..8894b48417c 100644 --- a/lib/gitlab/sidekiq_logging/json_formatter.rb +++ b/lib/gitlab/sidekiq_logging/json_formatter.rb @@ -18,10 +18,15 @@ module Gitlab when String output[:message] = data when Hash - convert_to_iso8601!(data) - convert_retry_to_integer!(data) - stringify_args!(data) output.merge!(data) + + # jobstr is redundant and can include information we wanted to + # exclude (like arguments) + output.delete(:jobstr) + + convert_to_iso8601!(output) + convert_retry_to_integer!(output) + process_args!(output) end output.to_json + "\n" @@ -56,8 +61,11 @@ module Gitlab end end - def stringify_args!(payload) - payload['args'] = Gitlab::Utils::LogLimitedArray.log_limited_array(payload['args'].map(&:to_s)) if payload['args'] + def process_args!(payload) + return unless payload['args'] + + payload['args'] = Gitlab::ErrorTracking::Processor::SidekiqProcessor + .loggable_arguments(payload['args'], payload['class']) end end end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 4e39120f8a7..eb845c5ff8d 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -27,7 +27,7 @@ module Gitlab private def add_instrumentation_keys!(job, output_payload) - output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper::KEYS)) + output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper.keys)) end def add_logging_extras!(job, output_payload) @@ -36,6 +36,10 @@ module Gitlab ) end + def add_db_counters!(job, output_payload) + output_payload.merge!(job.slice(*::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS)) + end + def log_job_start(payload) payload['message'] = "#{base_message(payload)}: start" payload['job_status'] = 'start' @@ -50,6 +54,7 @@ module Gitlab payload = payload.dup add_instrumentation_keys!(job, payload) add_logging_extras!(job, payload) + add_db_counters!(job, payload) elapsed_time = elapsed(started_time) add_time_keys!(elapsed_time, payload) diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb index ddd1b91410b..bb0c18735bb 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/client.rb @@ -5,9 +5,6 @@ module Gitlab module DuplicateJobs class Client def call(worker_class, job, queue, _redis_pool, &block) - # We don't try to deduplicate jobs that are scheduled in the future - return yield if job['at'] - DuplicateJob.new(job, queue).schedule(&block) end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index fa742d07af2..0dc53c61e84 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -18,13 +18,13 @@ module Gitlab # When new jobs can be scheduled again, the strategy calls `#delete`. class DuplicateJob DUPLICATE_KEY_TTL = 6.hours + DEFAULT_STRATEGY = :until_executing attr_reader :existing_jid - def initialize(job, queue_name, strategy: :until_executing) + def initialize(job, queue_name) @job = job @queue_name = queue_name - @strategy = strategy end # This will continue the middleware chain if the job should be scheduled @@ -41,12 +41,12 @@ module Gitlab end # This method will return the jid that was set in redis - def check! + def check!(expiry = DUPLICATE_KEY_TTL) read_jid = nil Sidekiq.redis do |redis| redis.multi do |multi| - redis.set(idempotency_key, jid, ex: DUPLICATE_KEY_TTL, nx: true) + redis.set(idempotency_key, jid, ex: expiry, nx: true) read_jid = redis.get(idempotency_key) end end @@ -60,6 +60,10 @@ module Gitlab end end + def scheduled? + scheduled_at.present? + end + def duplicate? raise "Call `#check!` first to check for existing duplicates" unless existing_jid @@ -67,14 +71,36 @@ module Gitlab end def droppable? - idempotent? && duplicate? && ::Feature.disabled?("disable_#{queue_name}_deduplication") + idempotent? && ::Feature.disabled?("disable_#{queue_name}_deduplication") + end + + def scheduled_at + job['at'] + end + + def options + return {} unless worker_klass + return {} unless worker_klass.respond_to?(:get_deduplication_options) + + worker_klass.get_deduplication_options end private - attr_reader :queue_name, :strategy, :job + attr_reader :queue_name, :job attr_writer :existing_jid + def worker_klass + @worker_klass ||= worker_class_name.to_s.safe_constantize + end + + def strategy + return DEFAULT_STRATEGY unless worker_klass + return DEFAULT_STRATEGY unless worker_klass.respond_to?(:idempotent?) + + worker_klass.get_deduplicate_strategy + end + def worker_class_name job['class'] end @@ -104,11 +130,10 @@ module Gitlab end def idempotent? - worker_class = worker_class_name.to_s.safe_constantize - return false unless worker_class - return false unless worker_class.respond_to?(:idempotent?) + return false unless worker_klass + return false unless worker_klass.respond_to?(:idempotent?) - worker_class.idempotent? + worker_klass.idempotent? end end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb index 674e436b714..0ed4912c4cc 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb @@ -13,13 +13,13 @@ module Gitlab end def schedule(job) - if duplicate_job.check! && duplicate_job.duplicate? + if deduplicatable_job? && check! && duplicate_job.duplicate? job['duplicate-of'] = duplicate_job.existing_jid - end - if duplicate_job.droppable? - Gitlab::SidekiqLogging::DeduplicationLogger.instance.log(job, "dropped until executing") - return false + if duplicate_job.droppable? + Gitlab::SidekiqLogging::DeduplicationLogger.instance.log(job, "dropped until executing") + return false + end end yield @@ -34,6 +34,22 @@ module Gitlab private attr_reader :duplicate_job + + def deduplicatable_job? + !duplicate_job.scheduled? || duplicate_job.options[:including_scheduled] + end + + def check! + duplicate_job.check!(expiry) + end + + def expiry + return DuplicateJob::DUPLICATE_KEY_TTL unless duplicate_job.scheduled? + + time_diff = duplicate_job.scheduled_at.to_i - Time.now.to_i + + time_diff > 0 ? time_diff : DuplicateJob::DUPLICATE_KEY_TTL + end end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 61ed2fe1a06..6a942a6ce06 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -47,6 +47,10 @@ module Gitlab @metrics[:sidekiq_jobs_completion_seconds].observe(labels, monotonic_time) @metrics[:sidekiq_jobs_db_seconds].observe(labels, ActiveRecord::LogSubscriber.runtime / 1000) @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(job)) + @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(job)) + @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(job)) + @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(job)) + @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(job)) end end @@ -54,15 +58,19 @@ module Gitlab def init_metrics { - sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), - sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), - sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), - sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all) + sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds of cpu time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS), + sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), + sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), + sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all) } end @@ -70,6 +78,22 @@ module Gitlab defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 end + def get_redis_time(job) + job.fetch(:redis_duration_s, 0) + end + + def get_redis_calls(job) + job.fetch(:redis_calls, 0) + end + + def get_elasticsearch_time(job) + job.fetch(:elasticsearch_duration_s, 0) + end + + def get_elasticsearch_calls(job) + job.fetch(:elasticsearch_calls, 0) + end + def get_gitaly_time(job) job.fetch(:gitaly_duration_s, 0) end diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index 342dae456a8..2d8df2ca204 100644 --- a/lib/gitlab/slash_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -21,7 +21,7 @@ module Gitlab This chatops integration does not have any commands that can be executed. - #{footer} + #{help_footer} MESSAGE end diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb index 9911f9e62a6..1d253ca90f3 100644 --- a/lib/gitlab/snippet_search_results.rb +++ b/lib/gitlab/snippet_search_results.rb @@ -11,7 +11,7 @@ module Gitlab @query = query end - def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE) + def objects(scope, page: nil, per_page: DEFAULT_PER_PAGE, preload_method: nil) paginated_objects(snippet_titles, page, per_page) end diff --git a/lib/gitlab/sourcegraph.rb b/lib/gitlab/sourcegraph.rb index d0f12c8364a..ec404ebd309 100644 --- a/lib/gitlab/sourcegraph.rb +++ b/lib/gitlab/sourcegraph.rb @@ -19,7 +19,7 @@ module Gitlab private def feature - Feature.get(:sourcegraph) + Feature.get(:sourcegraph) # rubocop:disable Gitlab/AvoidFeatureGet end end end diff --git a/lib/gitlab/suggestions/commit_message.rb b/lib/gitlab/suggestions/commit_message.rb new file mode 100644 index 00000000000..d59a8fc3730 --- /dev/null +++ b/lib/gitlab/suggestions/commit_message.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Suggestions + class CommitMessage + DEFAULT_SUGGESTION_COMMIT_MESSAGE = + 'Apply %{suggestions_count} suggestion(s) to %{files_count} file(s)' + + def initialize(user, suggestion_set) + @user = user + @suggestion_set = suggestion_set + end + + def message + project = suggestion_set.project + user_defined_message = project.suggestion_commit_message.presence + message = user_defined_message || DEFAULT_SUGGESTION_COMMIT_MESSAGE + + Gitlab::StringPlaceholderReplacer + .replace_string_placeholders(message, PLACEHOLDERS_REGEX) do |key| + PLACEHOLDERS[key].call(user, suggestion_set) + end + end + + def self.format_paths(paths) + paths.sort.join(', ') + end + + private_class_method :format_paths + + private + + attr_reader :user, :suggestion_set + + PLACEHOLDERS = { + 'branch_name' => ->(user, suggestion_set) { suggestion_set.branch }, + 'files_count' => ->(user, suggestion_set) { suggestion_set.file_paths.length }, + 'file_paths' => ->(user, suggestion_set) { format_paths(suggestion_set.file_paths) }, + 'project_name' => ->(user, suggestion_set) { suggestion_set.project.name }, + 'project_path' => ->(user, suggestion_set) { suggestion_set.project.path }, + 'user_full_name' => ->(user, suggestion_set) { user.name }, + 'username' => ->(user, suggestion_set) { user.username }, + 'suggestions_count' => ->(user, suggestion_set) { suggestion_set.suggestions.size } + }.freeze + + # This regex is built dynamically using the keys from the PLACEHOLDER struct. + # So, we can easily add new placeholder just by modifying the PLACEHOLDER hash. + # This regex will build the new PLACEHOLDER_REGEX with the new information + PLACEHOLDERS_REGEX = Regexp.union(PLACEHOLDERS.keys.map do |key| + Regexp.new(Regexp.escape(key)) + end).freeze + end + end +end diff --git a/lib/gitlab/suggestions/file_suggestion.rb b/lib/gitlab/suggestions/file_suggestion.rb new file mode 100644 index 00000000000..73b9800f0b8 --- /dev/null +++ b/lib/gitlab/suggestions/file_suggestion.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +module Gitlab + module Suggestions + class FileSuggestion + include Gitlab::Utils::StrongMemoize + + SuggestionForDifferentFileError = Class.new(StandardError) + + def initialize + @suggestions = [] + end + + def add_suggestion(new_suggestion) + if for_different_file?(new_suggestion) + raise SuggestionForDifferentFileError, + 'Only add suggestions for the same file.' + end + + suggestions << new_suggestion + end + + def line_conflict? + strong_memoize(:line_conflict) do + _line_conflict? + end + end + + def new_content + @new_content ||= _new_content + end + + def file_path + @file_path ||= _file_path + end + + private + + attr_accessor :suggestions + + def blob + first_suggestion&.diff_file&.new_blob + end + + def blob_data_lines + blob.load_all_data! + blob.data.lines + end + + def current_content + @current_content ||= blob.nil? ? [''] : blob_data_lines + end + + def _new_content + current_content.tap do |content| + suggestions.each do |suggestion| + range = line_range(suggestion) + content[range] = suggestion.to_content + end + end.join + end + + def line_range(suggestion) + suggestion.from_line_index..suggestion.to_line_index + end + + def for_different_file?(suggestion) + file_path && file_path != suggestion_file_path(suggestion) + end + + def suggestion_file_path(suggestion) + suggestion&.diff_file&.file_path + end + + def first_suggestion + suggestions.first + end + + def _file_path + suggestion_file_path(first_suggestion) + end + + def _line_conflict? + has_conflict = false + + suggestions.each_with_object([]) do |suggestion, ranges| + range_in_test = line_range(suggestion) + + if has_range_conflict?(range_in_test, ranges) + has_conflict = true + break + end + + ranges << range_in_test + end + + has_conflict + end + + def has_range_conflict?(range_in_test, ranges) + ranges.any? do |range| + range.overlaps?(range_in_test) + end + end + end + end +end diff --git a/lib/gitlab/suggestions/suggestion_set.rb b/lib/gitlab/suggestions/suggestion_set.rb new file mode 100644 index 00000000000..22abef98bf0 --- /dev/null +++ b/lib/gitlab/suggestions/suggestion_set.rb @@ -0,0 +1,120 @@ +# frozen_string_literal: true + +module Gitlab + module Suggestions + class SuggestionSet + attr_reader :suggestions + + def initialize(suggestions) + @suggestions = suggestions + end + + def project + first_suggestion.project + end + + def branch + first_suggestion.branch + end + + def valid? + error_message.nil? + end + + def error_message + @error_message ||= _error_message + end + + def actions + @actions ||= suggestions_per_file.map do |file_path, file_suggestion| + { + action: 'update', + file_path: file_path, + content: file_suggestion.new_content + } + end + end + + def file_paths + @file_paths ||= suggestions.map(&:file_path).uniq + end + + private + + def first_suggestion + suggestions.first + end + + def suggestions_per_file + @suggestions_per_file ||= _suggestions_per_file + end + + def _suggestions_per_file + suggestions.each_with_object({}) do |suggestion, result| + file_path = suggestion.diff_file.file_path + file_suggestion = result[file_path] ||= FileSuggestion.new + file_suggestion.add_suggestion(suggestion) + end + end + + def file_suggestions + suggestions_per_file.values + end + + def first_file_suggestion + file_suggestions.first + end + + def _error_message + suggestions.each do |suggestion| + message = error_for_suggestion(suggestion) + + return message if message + end + + has_line_conflict = file_suggestions.any? do |file_suggestion| + file_suggestion.line_conflict? + end + + if has_line_conflict + return _('Suggestions are not applicable as their lines cannot overlap.') + end + + nil + end + + def error_for_suggestion(suggestion) + unless suggestion.diff_file + return _('A file was not found.') + end + + unless on_same_branch?(suggestion) + return _('Suggestions must all be on the same branch.') + end + + unless suggestion.appliable?(cached: false) + return _('A suggestion is not applicable.') + end + + unless latest_source_head?(suggestion) + return _('A file has been changed.') + end + + nil + end + + def on_same_branch?(suggestion) + branch == suggestion.branch + end + + # Checks whether the latest source branch HEAD matches with + # the position HEAD we're using to update the file content. Since + # the persisted HEAD is updated async (for MergeRequest), + # it's more consistent to fetch this data directly from the + # repository. + def latest_source_head?(suggestion) + suggestion.position.head_sha == suggestion.noteable.source_branch_sha + end + end + end +end diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb index 5992f24f4e9..72783a2d682 100644 --- a/lib/gitlab/themes.rb +++ b/lib/gitlab/themes.rb @@ -23,7 +23,8 @@ module Gitlab Theme.new(9, 'Red', 'ui-red'), Theme.new(10, 'Light Red', 'ui-light-red'), Theme.new(2, 'Dark', 'ui-dark'), - Theme.new(3, 'Light', 'ui-light') + Theme.new(3, 'Light', 'ui-light'), + Theme.new(11, 'Dark Mode (alpha)', 'gl-dark') ].freeze # Convenience method to get a space-separated String of all the theme diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 329f87d8be8..cd15130cee6 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -41,7 +41,7 @@ module Gitlab when Wiki wiki_url(object, **options) when WikiPage - instance.project_wiki_url(object.wiki.project, object.slug, **options) + wiki_page_url(object.wiki, object, **options) when ::DesignManagement::Design design_url(object, **options) else @@ -78,12 +78,21 @@ module Gitlab end end - def wiki_url(object, **options) - if object.container.is_a?(Project) - instance.project_wiki_url(object.container, Wiki::HOMEPAGE, **options) - else - raise NotImplementedError.new("No URL builder defined for #{object.inspect}") - end + def wiki_url(wiki, **options) + return wiki_page_url(wiki, Wiki::HOMEPAGE, **options) unless options[:action] + + options[:controller] = 'projects/wikis' + options[:namespace_id] = wiki.container.namespace + options[:project_id] = wiki.container + + instance.url_for(**options) + end + + def wiki_page_url(wiki, page, **options) + options[:action] ||= :show + options[:id] = page + + wiki_url(wiki, **options) end def design_url(design, **options) diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index e60c786b52c..7b6f5e69ee1 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -1,27 +1,25 @@ # frozen_string_literal: true -# For hardening usage ping and make it easier to add measures there is in place -# * alt_usage_data method -# handles StandardError and fallbacks into -1 this way not all measures fail if we encounter one exception +# When developing usage data metrics use the below usage data interface methods +# unless you have good reasons to implement custom usage data +# See `lib/gitlab/utils/usage_data.rb` # -# Examples: -# alt_usage_data { Gitlab::VERSION } -# alt_usage_data { Gitlab::CurrentSettings.uuid } -# -# * redis_usage_data method -# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent -# returns -1 when a block is sent or hash with all values -1 when a counter is sent -# different behaviour due to 2 different implementations of redis counter -# -# Examples: -# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) -# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } +# Examples +# issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), +# active_user_count: count(User.active) +# alt_usage_data { Gitlab::VERSION } +# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) +# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } + module Gitlab class UsageData BATCH_SIZE = 100 - FALLBACK = -1 class << self + include Gitlab::Utils::UsageData + include Gitlab::Utils::StrongMemoize + include Gitlab::UsageDataConcerns::Topology + def data(force_refresh: false) Rails.cache.fetch('usage_data', force: force_refresh, expires_in: 2.weeks) do uncached_data @@ -29,12 +27,15 @@ module Gitlab end def uncached_data + clear_memoized_limits + license_usage_data .merge(system_usage_data) .merge(features_usage_data) .merge(components_usage_data) .merge(cycle_analytics_usage_data) .merge(object_store_usage_data) + .merge(topology_usage_data) .merge(recording_ce_finish_data) end @@ -44,7 +45,7 @@ module Gitlab def license_usage_data { - recorded_at: Time.now, # should be calculated very first + recorded_at: recorded_at, uuid: alt_usage_data { Gitlab::CurrentSettings.uuid }, hostname: alt_usage_data { Gitlab.config.gitlab.host }, version: alt_usage_data { Gitlab::VERSION }, @@ -54,6 +55,10 @@ module Gitlab } end + def recorded_at + Time.now + end + def recording_ce_finish_data { recording_ce_finished_at: Time.now @@ -64,6 +69,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def system_usage_data alert_bot_incident_count = count(::Issue.authored(::User.alert_bot)) + issues_created_manually_from_alerts = count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot)) { counts: { @@ -114,7 +120,9 @@ module Gitlab issues_with_associated_zoom_link: count(ZoomMeeting.added_to_issue), issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), issues_with_embedded_grafana_charts_approx: grafana_embed_usage_data, - issues_created_gitlab_alerts: count(Issue.with_alert_management_alerts.not_authored_by(::User.alert_bot)), + issues_created_from_alerts: total_alert_issues, + issues_created_gitlab_alerts: issues_created_manually_from_alerts, + issues_created_manually_from_alerts: issues_created_manually_from_alerts, incident_issues: alert_bot_incident_count, alert_bot_incident_issues: alert_bot_incident_count, incident_labeled_issues: count(::Issue.with_label_attributes(IncidentManagement::CreateIssueService::INCIDENT_LABEL)), @@ -131,11 +139,17 @@ module Gitlab projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), projects_with_alerts_service_enabled: count(AlertsService.active), projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id), + projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id), + projects_with_terraform_states: distinct_count(::Terraform::State, :project_id), protected_branches: count(ProtectedBranch), releases: count(Release), remote_mirrors: count(RemoteMirror), snippets: count(Snippet), + personal_snippets: count(PersonalSnippet), + project_snippets: count(ProjectSnippet), suggestions: count(Suggestion), + terraform_reports: count(::Ci::JobArtifact.terraform_reports), + terraform_states: count(::Terraform::State), todos: count(Todo), uploads: count(Upload), web_hooks: count(WebHook), @@ -147,7 +161,8 @@ module Gitlab usage_counters, user_preferences_usage, ingress_modsecurity_usage, - container_expiration_policies_usage + container_expiration_policies_usage, + merge_requests_usage(default_time_period) ) } end @@ -174,19 +189,20 @@ module Gitlab def features_usage_data_ce { - container_registry_enabled: alt_usage_data { Gitlab.config.registry.enabled }, + instance_auto_devops_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.auto_devops_enabled? }, + container_registry_enabled: alt_usage_data(fallback: nil) { Gitlab.config.registry.enabled }, dependency_proxy_enabled: Gitlab.config.try(:dependency_proxy)&.enabled, - gitlab_shared_runners_enabled: alt_usage_data { Gitlab.config.gitlab_ci.shared_runners_enabled }, - gravatar_enabled: alt_usage_data { Gitlab::CurrentSettings.gravatar_enabled? }, - ldap_enabled: alt_usage_data { Gitlab.config.ldap.enabled }, - mattermost_enabled: alt_usage_data { Gitlab.config.mattermost.enabled }, - omniauth_enabled: alt_usage_data { Gitlab::Auth.omniauth_enabled? }, - prometheus_metrics_enabled: alt_usage_data { Gitlab::Metrics.prometheus_metrics_enabled? }, - reply_by_email_enabled: alt_usage_data { Gitlab::IncomingEmail.enabled? }, - signup_enabled: alt_usage_data { Gitlab::CurrentSettings.allow_signup? }, - web_ide_clientside_preview_enabled: alt_usage_data { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? }, + gitlab_shared_runners_enabled: alt_usage_data(fallback: nil) { Gitlab.config.gitlab_ci.shared_runners_enabled }, + gravatar_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gravatar_enabled? }, + ldap_enabled: alt_usage_data(fallback: nil) { Gitlab.config.ldap.enabled }, + mattermost_enabled: alt_usage_data(fallback: nil) { Gitlab.config.mattermost.enabled }, + omniauth_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth.omniauth_enabled? }, + prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? }, + reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::IncomingEmail.enabled? }, + signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, + web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.web_ide_clientside_preview_enabled? }, ingress_modsecurity_enabled: Feature.enabled?(:ingress_modsecurity), - grafana_link_enabled: alt_usage_data { Gitlab::CurrentSettings.grafana_enabled? } + grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? } } end @@ -213,14 +229,15 @@ module Gitlab def components_usage_data { - git: { version: alt_usage_data { Gitlab::Git.version } }, + git: { version: alt_usage_data(fallback: { major: -1 }) { Gitlab::Git.version } }, gitaly: { version: alt_usage_data { Gitaly::Server.all.first.server_version }, servers: alt_usage_data { Gitaly::Server.count }, - filesystems: alt_usage_data { Gitaly::Server.filesystems } + clusters: alt_usage_data { Gitaly::Server.gitaly_clusters }, + filesystems: alt_usage_data(fallback: ["-1"]) { Gitaly::Server.filesystems } }, gitlab_pages: { - enabled: alt_usage_data { Gitlab.config.pages.enabled }, + enabled: alt_usage_data(fallback: nil) { Gitlab.config.pages.enabled }, version: alt_usage_data { Gitlab::Pages::VERSION } }, database: { @@ -382,65 +399,67 @@ module Gitlab {} # augmented in EE end - def count(relation, column = nil, batch: true, start: nil, finish: nil) - if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true) - Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish) - else - relation.count - end - rescue ActiveRecord::StatementInvalid - FALLBACK - end + # rubocop: disable CodeReuse/ActiveRecord + def merge_requests_usage(time_period) + query = + Event + .where(target_type: Event::TARGET_TYPES[:merge_request].to_s) + .where(time_period) + + merge_request_users = distinct_count( + query, + :author_id, + batch_size: 5_000, # Based on query performance, this is the optimal batch size. + start: User.minimum(:id), + finish: User.maximum(:id) + ) - def distinct_count(relation, column = nil, batch: true, start: nil, finish: nil) - if batch && Feature.enabled?(:usage_ping_batch_counter, default_enabled: true) - Gitlab::Database::BatchCount.batch_distinct_count(relation, column, start: start, finish: finish) - else - relation.distinct_count_by(column) - end - rescue ActiveRecord::StatementInvalid - FALLBACK + { + merge_requests_users: merge_request_users + } end + # rubocop: enable CodeReuse/ActiveRecord - def alt_usage_data(value = nil, fallback: FALLBACK, &block) - if block_given? - yield + def installation_type + if Rails.env.production? + Gitlab::INSTALLATION_TYPE else - value + "gitlab-development-kit" end - rescue - fallback end - def redis_usage_data(counter = nil, &block) - if block_given? - redis_usage_counter(&block) - elsif counter.present? - redis_usage_data_totals(counter) - end + def default_time_period + { created_at: 28.days.ago..Time.current } end private - def redis_usage_counter - yield - rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent - FALLBACK + def total_alert_issues + # Remove prometheus table queries once they are deprecated + # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407. + [ + count(Issue.with_alert_management_alerts), + count(::Issue.with_self_managed_prometheus_alert_events), + count(::Issue.with_prometheus_alert_events) + ].reduce(:+) end - def redis_usage_data_totals(counter) - counter.totals - rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent - counter.fallback_totals + def user_minimum_id + strong_memoize(:user_minimum_id) do + ::User.minimum(:id) + end end - def installation_type - if Rails.env.production? - Gitlab::INSTALLATION_TYPE - else - "gitlab-development-kit" + def user_maximum_id + strong_memoize(:user_maximum_id) do + ::User.maximum(:id) end end + + def clear_memoized_limits + clear_memoization(:user_minimum_id) + clear_memoization(:user_maximum_id) + end end end end diff --git a/lib/gitlab/usage_data_concerns/topology.rb b/lib/gitlab/usage_data_concerns/topology.rb new file mode 100644 index 00000000000..6e1d29f2a17 --- /dev/null +++ b/lib/gitlab/usage_data_concerns/topology.rb @@ -0,0 +1,137 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataConcerns + module Topology + include Gitlab::Utils::UsageData + + JOB_TO_SERVICE_NAME = { + 'gitlab-rails' => 'web', + 'gitlab-sidekiq' => 'sidekiq', + 'gitlab-workhorse' => 'workhorse', + 'redis' => 'redis', + 'postgres' => 'postgres', + 'gitaly' => 'gitaly', + 'prometheus' => 'prometheus', + 'node' => 'node-exporter' + }.freeze + + def topology_usage_data + topology_data, duration = measure_duration do + alt_usage_data(fallback: {}) do + { + nodes: topology_node_data + }.compact + end + end + { topology: topology_data.merge(duration_s: duration) } + end + + private + + def topology_node_data + with_prometheus_client do |client| + # node-level data + by_instance_mem = topology_node_memory(client) + by_instance_cpus = topology_node_cpus(client) + # service-level data + by_instance_by_job_by_metric_memory = topology_all_service_memory(client) + by_instance_by_job_process_count = topology_all_service_process_count(client) + + instances = Set.new(by_instance_mem.keys + by_instance_cpus.keys) + instances.map do |instance| + { + node_memory_total_bytes: by_instance_mem[instance], + node_cpus: by_instance_cpus[instance], + node_services: + topology_node_services(instance, by_instance_by_job_process_count, by_instance_by_job_by_metric_memory) + }.compact + end + end + end + + def topology_node_memory(client) + aggregate_single(client, 'avg (node_memory_MemTotal_bytes) by (instance)') + end + + def topology_node_cpus(client) + aggregate_single(client, 'count (node_cpu_seconds_total{mode="idle"}) by (instance)') + end + + def topology_all_service_memory(client) + aggregate_many( + client, + 'avg ({__name__ =~ "(ruby_){0,1}process_(resident|unique|proportional)_memory_bytes", job != "gitlab_exporter_process"}) by (instance, job, __name__)' + ) + end + + def topology_all_service_process_count(client) + aggregate_many(client, 'count ({__name__ =~ "(ruby_){0,1}process_start_time_seconds", job != "gitlab_exporter_process"}) by (instance, job)') + end + + def topology_node_services(instance, all_process_counts, all_process_memory) + # returns all node service data grouped by service name as the key + instance_service_data = + topology_instance_service_process_count(instance, all_process_counts) + .deep_merge(topology_instance_service_memory(instance, all_process_memory)) + + # map to list of hashes where service names become values instead, and remove + # unknown services, since they might not be ours + instance_service_data.each_with_object([]) do |entry, list| + service, service_metrics = entry + gitlab_service = JOB_TO_SERVICE_NAME[service.to_s] + next unless gitlab_service + + list << { name: gitlab_service }.merge(service_metrics) + end + end + + def topology_instance_service_process_count(instance, all_instance_data) + topology_data_for_instance(instance, all_instance_data).to_h do |metric, count| + [metric['job'], { process_count: count }] + end + end + + def topology_instance_service_memory(instance, all_instance_data) + topology_data_for_instance(instance, all_instance_data).each_with_object({}) do |entry, hash| + metric, memory = entry + job = metric['job'] + key = + case metric['__name__'] + when match_process_memory_metric_for_type('resident') then :process_memory_rss + when match_process_memory_metric_for_type('unique') then :process_memory_uss + when match_process_memory_metric_for_type('proportional') then :process_memory_pss + end + + hash[job] ||= {} + hash[job][key] ||= memory + end + end + + def match_process_memory_metric_for_type(type) + /(ruby_){0,1}process_#{type}_memory_bytes/ + end + + def topology_data_for_instance(instance, all_instance_data) + all_instance_data.filter { |metric, _value| metric['instance'] == instance } + end + + def drop_port(instance) + instance.gsub(/:.+$/, '') + end + + # Will retain a single `instance` key that values are mapped to + def aggregate_single(client, query) + client.aggregate(query) { |metric| drop_port(metric['instance']) } + end + + # Will retain a composite key that values are mapped to + def aggregate_many(client, query) + client.aggregate(query) do |metric| + metric['instance'] = drop_port(metric['instance']) + metric + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb index 96898e5189c..44893645cc2 100644 --- a/lib/gitlab/usage_data_counters/base_counter.rb +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -8,7 +8,7 @@ module Gitlab::UsageDataCounters class << self def redis_key(event) - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new, event: event) unless known_events.include?(event.to_s) + require_known_event(event) "USAGE_#{prefix}_#{event}".upcase end @@ -31,6 +31,10 @@ module Gitlab::UsageDataCounters private + def require_known_event(event) + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new, event: event) unless known_events.include?(event.to_s) + end + def counter_key(event) "#{prefix}_#{event}".to_sym end diff --git a/lib/gitlab/usage_data_counters/designs_counter.rb b/lib/gitlab/usage_data_counters/designs_counter.rb index 801fb8f3b3d..22188b555d2 100644 --- a/lib/gitlab/usage_data_counters/designs_counter.rb +++ b/lib/gitlab/usage_data_counters/designs_counter.rb @@ -4,7 +4,7 @@ module Gitlab::UsageDataCounters class DesignsCounter extend Gitlab::UsageDataCounters::RedisCounter - KNOWN_EVENTS = %w[create update delete].map(&:freeze).freeze + KNOWN_EVENTS = %w[create update delete].freeze UnknownEvent = Class.new(StandardError) diff --git a/lib/gitlab/usage_data_counters/search_counter.rb b/lib/gitlab/usage_data_counters/search_counter.rb index b9e3a5c0104..61f98887adc 100644 --- a/lib/gitlab/usage_data_counters/search_counter.rb +++ b/lib/gitlab/usage_data_counters/search_counter.rb @@ -2,28 +2,20 @@ module Gitlab module UsageDataCounters - class SearchCounter - extend RedisCounter - - NAVBAR_SEARCHES_COUNT_KEY = 'NAVBAR_SEARCHES_COUNT' + class SearchCounter < BaseCounter + KNOWN_EVENTS = %w[all_searches navbar_searches].freeze class << self - def increment_navbar_searches_count - increment(NAVBAR_SEARCHES_COUNT_KEY) - end + def redis_key(event) + require_known_event(event) - def total_navbar_searches_count - total_count(NAVBAR_SEARCHES_COUNT_KEY) + "#{event}_COUNT".upcase end - def totals - { - navbar_searches: total_navbar_searches_count - } - end + private - def fallback_totals - { navbar_searches: -1 } + def counter_key(event) + "#{event}".to_sym end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index d46601fa2e8..e80cc51dc3b 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -3,6 +3,7 @@ module Gitlab module Utils extend self + PathTraversalAttackError ||= Class.new(StandardError) # Ensure that the relative path will not traverse outside the base directory # We url decode the path to avoid passing invalid paths forward in url encoded format. @@ -17,7 +18,7 @@ module Gitlab path.end_with?("#{File::SEPARATOR}..") || (!allowed_absolute && Pathname.new(path).absolute?) - raise StandardError.new("Invalid path") + raise PathTraversalAttackError.new('Invalid path') end path @@ -159,5 +160,23 @@ module Gitlab Addressable::URI.parse(uri_string) rescue Addressable::URI::InvalidURIError, TypeError end + + # Invert a hash, collecting all keys that map to a given value in an array. + # + # Unlike `Hash#invert`, where the last encountered pair wins, and which has the + # type `Hash[k, v] => Hash[v, k]`, `multiple_key_invert` does not lose any + # information, has the type `Hash[k, v] => Hash[v, Array[k]]`, and the original + # hash can always be reconstructed. + # + # example: + # + # multiple_key_invert({ a: 1, b: 2, c: 1 }) + # # => { 1 => [:a, :c], 2 => [:b] } + # + def multiple_key_invert(hash) + hash.flat_map { |k, v| Array.wrap(v).zip([k].cycle) } + .group_by(&:first) + .transform_values { |kvs| kvs.map(&:last) } + end end end diff --git a/lib/gitlab/utils/log_limited_array.rb b/lib/gitlab/utils/log_limited_array.rb index e0589c3df4c..fbbba568d14 100644 --- a/lib/gitlab/utils/log_limited_array.rb +++ b/lib/gitlab/utils/log_limited_array.rb @@ -9,14 +9,14 @@ module Gitlab # to around 10 kilobytes. Once we hit the limit, add the sentinel # value as the last item in the returned array. def self.log_limited_array(array, sentinel: '...') - return [] unless array.is_a?(Array) + return [] unless array.is_a?(Array) || array.is_a?(Enumerator::Lazy) total_length = 0 limited_array = array.take_while do |arg| total_length += JsonSizeEstimator.estimate(arg) total_length <= MAXIMUM_ARRAY_LENGTH - end + end.to_a limited_array.push(sentinel) if total_length > MAXIMUM_ARRAY_LENGTH diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb new file mode 100644 index 00000000000..afc4e000977 --- /dev/null +++ b/lib/gitlab/utils/usage_data.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +# Usage data utilities +# +# * distinct_count(relation, column = nil, batch: true, start: nil, finish: nil) +# Does a distinct batch count, smartly reduces batch_size and handles errors +# +# Examples: +# issues_using_zoom_quick_actions: distinct_count(ZoomMeeting, :issue_id), +# +# * count(relation, column = nil, batch: true, start: nil, finish: nil) +# Does a non-distinct batch count, smartly reduces batch_size and handles errors +# +# Examples: +# active_user_count: count(User.active) +# +# * alt_usage_data method +# handles StandardError and fallbacks by default into -1 this way not all measures fail if we encounter one exception +# there might be cases where we need to set a specific fallback in order to be aligned wih what version app is expecting as a type +# +# Examples: +# alt_usage_data { Gitlab::VERSION } +# alt_usage_data { Gitlab::CurrentSettings.uuid } +# alt_usage_data(fallback: nil) { Gitlab.config.registry.enabled } +# +# * redis_usage_data method +# handles ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent +# returns -1 when a block is sent or hash with all values -1 when a counter is sent +# different behaviour due to 2 different implementations of redis counter +# +# Examples: +# redis_usage_data(Gitlab::UsageDataCounters::WikiPageCounter) +# redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } + +module Gitlab + module Utils + module UsageData + extend self + + FALLBACK = -1 + + def count(relation, column = nil, batch: true, start: nil, finish: nil) + if batch + Gitlab::Database::BatchCount.batch_count(relation, column, start: start, finish: finish) + else + relation.count + end + rescue ActiveRecord::StatementInvalid + FALLBACK + end + + def distinct_count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) + if batch + Gitlab::Database::BatchCount.batch_distinct_count(relation, column, batch_size: batch_size, start: start, finish: finish) + else + relation.distinct_count_by(column) + end + rescue ActiveRecord::StatementInvalid + FALLBACK + end + + def alt_usage_data(value = nil, fallback: FALLBACK, &block) + if block_given? + yield + else + value + end + rescue + fallback + end + + def redis_usage_data(counter = nil, &block) + if block_given? + redis_usage_counter(&block) + elsif counter.present? + redis_usage_data_totals(counter) + end + end + + def with_prometheus_client + if Gitlab::Prometheus::Internal.prometheus_enabled? + prometheus_address = Gitlab::Prometheus::Internal.uri + yield Gitlab::PrometheusClient.new(prometheus_address, allow_local_requests: true) + end + end + + def measure_duration + result = nil + duration = Benchmark.realtime do + result = yield + end + [result, duration] + end + + private + + def redis_usage_counter + yield + rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent + FALLBACK + end + + def redis_usage_data_totals(counter) + counter.totals + rescue ::Redis::CommandError, Gitlab::UsageDataCounters::BaseCounter::UnknownEvent + counter.fallback_totals + end + end + end +end diff --git a/lib/gitlab/web_ide/config.rb b/lib/gitlab/web_ide/config.rb new file mode 100644 index 00000000000..3b1fa162b53 --- /dev/null +++ b/lib/gitlab/web_ide/config.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module WebIde + # + # Base GitLab WebIde Configuration facade + # + class Config + ConfigError = Class.new(StandardError) + + def initialize(config, opts = {}) + @config = build_config(config, opts) + + @global = Entry::Global.new(@config, + with_image_ports: true) + @global.compose! + rescue Gitlab::Config::Loader::FormatError => e + raise Config::ConfigError, e.message + end + + def valid? + @global.valid? + end + + def errors + @global.errors + end + + def to_hash + @config + end + + def terminal_value + @global.terminal_value + end + + private + + def build_config(config, opts = {}) + Gitlab::Config::Loader::Yaml.new(config).load! + end + end + end +end diff --git a/lib/gitlab/web_ide/config/entry/global.rb b/lib/gitlab/web_ide/config/entry/global.rb new file mode 100644 index 00000000000..50c3f2d294f --- /dev/null +++ b/lib/gitlab/web_ide/config/entry/global.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module WebIde + class Config + module Entry + ## + # This class represents a global entry - root Entry for entire + # GitLab WebIde Configuration file. + # + class Global < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[terminal].freeze + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + end + + entry :terminal, Entry::Terminal, + description: 'Configuration of the webide terminal.' + + attributes :terminal + end + end + end + end +end diff --git a/lib/gitlab/web_ide/config/entry/terminal.rb b/lib/gitlab/web_ide/config/entry/terminal.rb new file mode 100644 index 00000000000..403e308d45b --- /dev/null +++ b/lib/gitlab/web_ide/config/entry/terminal.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module WebIde + class Config + module Entry + ## + # Entry that represents a concrete CI/CD job. + # + class Terminal < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Attributable + + # By default the build will finish in a few seconds, not giving the webide + # enough time to connect to the terminal. This default script provides + # those seconds blocking the build from finishing inmediately. + DEFAULT_SCRIPT = ['sleep 60'].freeze + + ALLOWED_KEYS = %i[image services tags before_script script variables].freeze + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + validates :config, job_port_unique: { data: ->(record) { record.ports } } + + with_options allow_nil: true do + validates :tags, array_of_strings: true + end + end + + entry :before_script, ::Gitlab::Ci::Config::Entry::Script, + description: 'Global before script overridden in this job.' + + entry :script, ::Gitlab::Ci::Config::Entry::Commands, + description: 'Commands that will be executed in this job.' + + entry :image, ::Gitlab::Ci::Config::Entry::Image, + description: 'Image that will be used to execute this job.' + + entry :services, ::Gitlab::Ci::Config::Entry::Services, + description: 'Services that will be used to execute this job.' + + entry :variables, ::Gitlab::Ci::Config::Entry::Variables, + description: 'Environment variables available for this job.' + + attributes :tags + + def value + to_hash.compact + end + + private + + def to_hash + { tag_list: tags || [], + yaml_variables: yaml_variables, + options: { + image: image_value, + services: services_value, + before_script: before_script_value, + script: script_value || DEFAULT_SCRIPT + }.compact } + end + + def yaml_variables + return unless variables_value + + variables_value.map do |key, value| + { key: key.to_s, value: value, public: true } + end + end + end + end + end + end +end |