diff options
Diffstat (limited to 'lib/gitlab')
242 files changed, 4849 insertions, 2108 deletions
diff --git a/lib/gitlab/alert_management/payload.rb b/lib/gitlab/alert_management/payload.rb index ce09ffd87ee..a1063001330 100644 --- a/lib/gitlab/alert_management/payload.rb +++ b/lib/gitlab/alert_management/payload.rb @@ -17,13 +17,14 @@ module Gitlab # @param project [Project] # @param payload [Hash] # @param monitoring_tool [String] - def parse(project, payload, monitoring_tool: nil) + # @param integration [AlertManagement::HttpIntegration] + def parse(project, payload, monitoring_tool: nil, integration: nil) payload_class = payload_class_for( monitoring_tool: monitoring_tool || payload&.dig('monitoring_tool'), payload: payload ) - payload_class.new(project: project, payload: payload) + payload_class.new(project: project, payload: payload, integration: integration) end private @@ -47,3 +48,5 @@ module Gitlab end end end + +Gitlab::AlertManagement::Payload.prepend_if_ee('EE::Gitlab::AlertManagement::Payload') diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index 0fd593a3780..c8b8d6c259d 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -12,7 +12,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize include Gitlab::Routing - attr_accessor :project, :payload + attr_accessor :project, :payload, :integration # Any attribute expected to be specifically read from # or derived from an alert payload should be defined. @@ -44,11 +44,25 @@ module Gitlab :title ].freeze + private_constant :EXPECTED_PAYLOAD_ATTRIBUTES + # Define expected API for a payload EXPECTED_PAYLOAD_ATTRIBUTES.each do |key| define_method(key) {} end + SEVERITY_MAPPING = { + 'critical' => :critical, + 'high' => :high, + 'medium' => :medium, + 'low' => :low, + 'info' => :info + }.freeze + + # Handle an unmapped severity value the same way we treat missing values + # so we can fallback to alert's default severity `critical`. + UNMAPPED_SEVERITY = nil + # Defines a method which allows access to a given # value within an alert payload # @@ -131,9 +145,21 @@ module Gitlab true end + def severity + severity_mapping.fetch(severity_raw.to_s.downcase, UNMAPPED_SEVERITY) + end + private - def plain_gitlab_fingerprint; end + def plain_gitlab_fingerprint + end + + def severity_raw + end + + def severity_mapping + SEVERITY_MAPPING + end def truncate_hosts(hosts) return hosts if hosts.join.length <= ::AlertManagement::Alert::HOSTS_MAX_LENGTH @@ -147,6 +173,7 @@ module Gitlab end end + # Overriden in EE::Gitlab::AlertManagement::Payload::Generic def value_for_paths(paths) target_path = paths.find { |path| payload&.dig(*path) } diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb index e8e85155bef..0eb1bee8181 100644 --- a/lib/gitlab/alert_management/payload/generic.rb +++ b/lib/gitlab/alert_management/payload/generic.rb @@ -6,7 +6,6 @@ module Gitlab module Payload class Generic < Base DEFAULT_TITLE = 'New: Incident' - DEFAULT_SEVERITY = 'critical' attribute :description, paths: 'description' attribute :ends_at, paths: 'end_time', type: :time @@ -15,10 +14,12 @@ module Gitlab attribute :monitoring_tool, paths: 'monitoring_tool' attribute :runbook, paths: 'runbook' attribute :service, paths: 'service' - attribute :severity, paths: 'severity', fallback: -> { DEFAULT_SEVERITY } attribute :starts_at, paths: 'start_time', type: :time, fallback: -> { Time.current.utc } attribute :title, paths: 'title', fallback: -> { DEFAULT_TITLE } + attribute :severity_raw, paths: 'severity' + private :severity_raw + attribute :plain_gitlab_fingerprint, paths: 'fingerprint' private :plain_gitlab_fingerprint end diff --git a/lib/gitlab/alert_management/payload/prometheus.rb b/lib/gitlab/alert_management/payload/prometheus.rb index 336e9b319e8..4c36ebbf3aa 100644 --- a/lib/gitlab/alert_management/payload/prometheus.rb +++ b/lib/gitlab/alert_management/payload/prometheus.rb @@ -1,10 +1,12 @@ # frozen_string_literal: true -# Attribute mapping for alerts via prometheus alerting integration. module Gitlab module AlertManagement module Payload + # Attribute mapping for alerts via prometheus alerting integration. class Prometheus < Base + extend Gitlab::Utils::Override + attribute :alert_markdown, paths: %w(annotations gitlab_incident_markdown) attribute :annotations, paths: 'annotations' attribute :description, paths: %w(annotations description) @@ -26,13 +28,40 @@ module Gitlab paths: [%w(annotations title), %w(annotations summary), %w(labels alertname)] - attribute :starts_at_raw, paths: [%w(startsAt)] private :starts_at_raw + attribute :severity_raw, paths: %w(labels severity) + private :severity_raw + METRIC_TIME_WINDOW = 30.minutes + ADDITIONAL_SEVERITY_MAPPING = { + 's1' => :critical, + 's2' => :high, + 's3' => :medium, + 's4' => :low, + 's5' => :info, + 'p1' => :critical, + 'p2' => :high, + 'p3' => :medium, + 'p4' => :low, + 'p5' => :info, + 'debug' => :info, + 'information' => :info, + 'notice' => :info, + 'warn' => :low, + 'warning' => :low, + 'minor' => :low, + 'error' => :medium, + 'major' => :high, + 'emergency' => :critical, + 'fatal' => :critical, + 'alert' => :medium, + 'page' => :high + }.freeze + def monitoring_tool Gitlab::AlertManagement::Payload::MONITORING_TOOLS[:prometheus] end @@ -67,6 +96,11 @@ module Gitlab private + override :severity_mapping + def severity_mapping + super.merge(ADDITIONAL_SEVERITY_MAPPING) + end + def plain_gitlab_fingerprint [starts_at_raw, title, full_query].join('/') end diff --git a/lib/gitlab/api_authentication/token_locator.rb b/lib/gitlab/api_authentication/token_locator.rb index 32a98908e5b..09039f3fc43 100644 --- a/lib/gitlab/api_authentication/token_locator.rb +++ b/lib/gitlab/api_authentication/token_locator.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :location - validates :location, inclusion: { in: %i[http_basic_auth] } + validates :location, inclusion: { in: %i[http_basic_auth http_token] } def initialize(location) @location = location @@ -21,6 +21,8 @@ module Gitlab case @location when :http_basic_auth extract_from_http_basic_auth request + when :http_token + extract_from_http_token request end end @@ -32,6 +34,13 @@ module Gitlab UsernameAndPassword.new(username, password) end + + def extract_from_http_token(request) + password = request.headers['Authorization'] + return unless password.present? + + UsernameAndPassword.new(nil, password) + end end end end diff --git a/lib/gitlab/api_authentication/token_resolver.rb b/lib/gitlab/api_authentication/token_resolver.rb index 5b30777b6ec..9234837cdf7 100644 --- a/lib/gitlab/api_authentication/token_resolver.rb +++ b/lib/gitlab/api_authentication/token_resolver.rb @@ -7,7 +7,16 @@ module Gitlab attr_reader :token_type - validates :token_type, inclusion: { in: %i[personal_access_token job_token deploy_token] } + validates :token_type, inclusion: { + in: %i[ + personal_access_token_with_username + job_token_with_username + deploy_token_with_username + personal_access_token + job_token + deploy_token + ] + } def initialize(token_type) @token_type = token_type @@ -38,49 +47,94 @@ module Gitlab when :deploy_token resolve_deploy_token raw + + when :personal_access_token_with_username + resolve_personal_access_token_with_username raw + + when :job_token_with_username + resolve_job_token_with_username raw + + when :deploy_token_with_username + resolve_deploy_token_with_username raw end end private - def resolve_personal_access_token(raw) - # Check if the password is a personal access token - pat = ::PersonalAccessToken.find_by_token(raw.password) - return unless pat + def resolve_personal_access_token_with_username(raw) + raise ::Gitlab::Auth::UnauthorizedError unless raw.username + + with_personal_access_token(raw) do |pat| + break unless pat - # Ensure that the username matches the token. This check is a subtle - # departure from the existing behavior of #find_personal_access_token_from_http_basic_auth. - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_435907856 - raise ::Gitlab::Auth::UnauthorizedError unless pat.user.username == raw.username + # Ensure that the username matches the token. This check is a subtle + # departure from the existing behavior of #find_personal_access_token_from_http_basic_auth. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_435907856 + raise ::Gitlab::Auth::UnauthorizedError unless pat.user.username == raw.username - pat + pat + end end - def resolve_job_token(raw) + def resolve_job_token_with_username(raw) # Only look for a job if the username is correct return if ::Gitlab::Auth::CI_JOB_USER != raw.username - job = ::Ci::AuthJobFinder.new(token: raw.password).execute + with_job_token(raw) do |job| + job + end + end - # Actively reject credentials with the username `gitlab-ci-token` if - # the password is not a valid job token. This replicates existing - # behavior of #find_user_from_job_token. - raise ::Gitlab::Auth::UnauthorizedError unless job + def resolve_deploy_token_with_username(raw) + with_deploy_token(raw) do |token| + break unless token + + # Ensure that the username matches the token. This check is a subtle + # departure from the existing behavior of #deploy_token_from_request. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_474826205 + raise ::Gitlab::Auth::UnauthorizedError unless token.username == raw.username - job + token + end + end + + def resolve_personal_access_token(raw) + with_personal_access_token(raw) do |pat| + pat + end + end + + def resolve_job_token(raw) + with_job_token(raw) do |job| + job + end end def resolve_deploy_token(raw) - # Check if the password is a deploy token + with_deploy_token(raw) do |token| + token + end + end + + def with_personal_access_token(raw, &block) + pat = ::PersonalAccessToken.find_by_token(raw.password) + return unless pat + + yield(pat) + end + + def with_deploy_token(raw, &block) token = ::DeployToken.active.find_by_token(raw.password) return unless token - # Ensure that the username matches the token. This check is a subtle - # departure from the existing behavior of #deploy_token_from_request. - # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_474826205 - raise ::Gitlab::Auth::UnauthorizedError unless token.username == raw.username + yield(token) + end + + def with_job_token(raw, &block) + job = ::Ci::AuthJobFinder.new(token: raw.password).execute + raise ::Gitlab::Auth::UnauthorizedError unless job - token + yield(job) end end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index fbba86d1253..0a69a9c503d 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -20,6 +20,7 @@ module Gitlab def rate_limits { issues_create: { threshold: -> { application_settings.issues_create_limit }, interval: 1.minute }, + notes_create: { threshold: -> { application_settings.notes_create_limit }, interval: 1.minute }, project_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute }, project_download_export: { threshold: -> { application_settings.project_download_export_limit }, interval: 1.minute }, project_repositories_archive: { threshold: 5, interval: 1.minute }, @@ -46,15 +47,17 @@ module Gitlab # @option scope [Array<ActiveRecord>] Array of ActiveRecord models to scope throttling to a specific request (e.g. per user per project) # @option threshold [Integer] Optional threshold value to override default one registered in `.rate_limits` # @option interval [Integer] Optional interval value to override default one registered in `.rate_limits` + # @option users_allowlist [Array<String>] Optional list of usernames to excepted from the limit. This param will only be functional if Scope includes a current user. # # @return [Boolean] Whether or not a request should be throttled - def throttled?(key, scope: nil, interval: nil, threshold: nil) + def throttled?(key, **options) return unless rate_limits[key] - threshold_value = threshold || threshold(key) + return if scoped_user_in_allowlist?(options) + threshold_value = options[:threshold] || threshold(key) threshold_value > 0 && - increment(key, scope, interval) > threshold_value + increment(key, options[:scope], options[:interval]) > threshold_value end # Increments the given cache key and increments the value by 1 with the @@ -140,6 +143,15 @@ module Gitlab def application_settings Gitlab::CurrentSettings.current_application_settings end + + def scoped_user_in_allowlist?(options) + return unless options[:users_allowlist].present? + + scoped_user = [options[:scope]].flatten.find { |s| s.is_a?(User) } + return unless scoped_user + + scoped_user.username.downcase.in?(options[:users_allowlist]) + end end end end diff --git a/lib/gitlab/auth/otp/session_enforcer.rb b/lib/gitlab/auth/otp/session_enforcer.rb deleted file mode 100644 index 8cc280756cc..00000000000 --- a/lib/gitlab/auth/otp/session_enforcer.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Auth - module Otp - class SessionEnforcer - OTP_SESSIONS_NAMESPACE = 'session:otp' - DEFAULT_EXPIRATION = 15.minutes.to_i - - def initialize(key) - @key = key - end - - def update_session - Gitlab::Redis::SharedState.with do |redis| - redis.setex(key_name, DEFAULT_EXPIRATION, true) - end - end - - def access_restricted? - Gitlab::Redis::SharedState.with do |redis| - !redis.get(key_name) - end - end - - private - - attr_reader :key - - def key_name - @key_name ||= "#{OTP_SESSIONS_NAMESPACE}:#{key.id}" - end - end - end - end -end diff --git a/lib/gitlab/auth/u2f_webauthn_converter.rb b/lib/gitlab/auth/u2f_webauthn_converter.rb new file mode 100644 index 00000000000..f85b2248aeb --- /dev/null +++ b/lib/gitlab/auth/u2f_webauthn_converter.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + class U2fWebauthnConverter + def initialize(u2f_registration) + @u2f_registration = u2f_registration + end + + def convert + now = Time.current + + converted_credential = WebAuthn::U2fMigrator.new( + app_id: Gitlab.config.gitlab.url, + certificate: u2f_registration.certificate, + key_handle: u2f_registration.key_handle, + public_key: u2f_registration.public_key, + counter: u2f_registration.counter + ).credential + + { + credential_xid: Base64.strict_encode64(converted_credential.id), + public_key: Base64.strict_encode64(converted_credential.public_key), + counter: u2f_registration.counter || 0, + name: u2f_registration.name || '', + user_id: u2f_registration.user_id, + u2f_registration_id: u2f_registration.id, + created_at: now, + updated_at: now + } + end + + private + + attr_reader :u2f_registration + end + end +end diff --git a/lib/gitlab/background_migration.rb b/lib/gitlab/background_migration.rb index d1b9062a23c..9f4d6557023 100644 --- a/lib/gitlab/background_migration.rb +++ b/lib/gitlab/background_migration.rb @@ -33,7 +33,7 @@ module Gitlab next unless job.queue == self.queue next unless migration_class == steal_class - next if block_given? && !(yield migration_args) + next if block_given? && !(yield job) begin perform(migration_class, migration_args) if job.delete diff --git a/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb new file mode 100644 index 00000000000..61eb3b332de --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_updated_at_after_repository_storage_move.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Update existent project update_at column after their repository storage was moved + class BackfillProjectUpdatedAtAfterRepositoryStorageMove + def perform(*project_ids) + updated_repository_storages = ProjectRepositoryStorageMove.select("project_id, MAX(updated_at) as updated_at").where(project_id: project_ids).group(:project_id) + + Project.connection.execute <<-SQL + WITH repository_storage_cte as ( + #{updated_repository_storages.to_sql} + ) + UPDATE projects + SET updated_at = (repository_storage_cte.updated_at + interval '1 second') + FROM repository_storage_cte + WHERE projects.id = repository_storage_cte.project_id AND projects.updated_at <= repository_storage_cte.updated_at + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb new file mode 100644 index 00000000000..de2d9909961 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_devops_segments_to_groups.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true +module Gitlab + module BackgroundMigration + # EE-specific migration + class MigrateDevopsSegmentsToGroups + def perform + # no-op for CE + end + end + end +end + +Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateDevopsSegmentsToGroups') diff --git a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb index b8c14aa2573..091e6660bac 100644 --- a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb +++ b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb @@ -16,26 +16,9 @@ module Gitlab def perform(start_id, end_id) old_registrations = U2fRegistration.where(id: start_id..end_id) old_registrations.each_slice(100) do |slice| - now = Time.now values = slice.map do |u2f_registration| - converted_credential = WebAuthn::U2fMigrator.new( - app_id: Gitlab.config.gitlab.url, - certificate: u2f_registration.certificate, - key_handle: u2f_registration.key_handle, - public_key: u2f_registration.public_key, - counter: u2f_registration.counter - ).credential - - { - credential_xid: Base64.strict_encode64(converted_credential.id), - public_key: Base64.strict_encode64(converted_credential.public_key), - counter: u2f_registration.counter || 0, - name: u2f_registration.name || '', - user_id: u2f_registration.user_id, - u2f_registration_id: u2f_registration.id, - created_at: now, - updated_at: now - } + converter = Gitlab::Auth::U2fWebauthnConverter.new(u2f_registration) + converter.convert end WebauthnRegistration.insert_all(values, unique_by: :credential_xid, returning: false) diff --git a/lib/gitlab/background_migration/populate_issue_email_participants.rb b/lib/gitlab/background_migration/populate_issue_email_participants.rb new file mode 100644 index 00000000000..d6795296fb7 --- /dev/null +++ b/lib/gitlab/background_migration/populate_issue_email_participants.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class to migrate service_desk_reply_to email addresses to issue_email_participants + class PopulateIssueEmailParticipants + # rubocop:disable Style/Documentation + class TmpIssue < ActiveRecord::Base + self.table_name = 'issues' + end + + def perform(start_id, stop_id) + issues = TmpIssue.select(:id, :service_desk_reply_to, :created_at).where(id: (start_id..stop_id)).where.not(service_desk_reply_to: nil) + + rows = issues.map do |issue| + { + issue_id: issue.id, + email: issue.service_desk_reply_to, + created_at: issue.created_at, + updated_at: issue.created_at + } + end + + Gitlab::Database.bulk_insert(:issue_email_participants, rows, on_conflict: :do_nothing) # rubocop:disable Gitlab/BulkInsert + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb new file mode 100644 index 00000000000..3d3970f50e1 --- /dev/null +++ b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop:disable Style/Documentation + class PopulateUuidsForSecurityFindings + NOP_RELATION = Class.new { def each_batch(*); end } + + def self.security_findings + NOP_RELATION.new + end + + def perform(_scan_ids); end + end + end +end + +Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_if_ee('::EE::Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings') diff --git a/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb new file mode 100644 index 00000000000..ca61118a06c --- /dev/null +++ b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +# rubocop: disable Style/Documentation +class Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings + DELETE_BATCH_SIZE = 100 + + # rubocop:disable Gitlab/NamespacedClass + class VulnerabilitiesFinding < ActiveRecord::Base + self.table_name = "vulnerability_occurrences" + end + # rubocop:enable Gitlab/NamespacedClass + + def perform(start_id, end_id) + batch = VulnerabilitiesFinding.where(id: start_id..end_id) + + cte = Gitlab::SQL::CTE.new(:batch, batch.select(:report_type, :location_fingerprint, :primary_identifier_id, :project_id)) + + query = VulnerabilitiesFinding + .select('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id', 'array_agg(id) as ids') + .distinct + .with(cte.to_arel) + .from(cte.alias_to(Arel.sql('batch'))) + .joins( + %( + INNER JOIN + vulnerability_occurrences ON + vulnerability_occurrences.report_type = batch.report_type AND + vulnerability_occurrences.location_fingerprint = batch.location_fingerprint AND + vulnerability_occurrences.primary_identifier_id = batch.primary_identifier_id AND + vulnerability_occurrences.project_id = batch.project_id + )).group('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id') + .having('COUNT(*) > 1') + + ids_to_delete = [] + + query.to_a.each do |record| + # We want to keep the latest finding since it might have recent metadata + duplicate_ids = record.ids.uniq.sort + duplicate_ids.pop + ids_to_delete.concat(duplicate_ids) + + if ids_to_delete.size == DELETE_BATCH_SIZE + VulnerabilitiesFinding.where(id: ids_to_delete).delete_all + ids_to_delete.clear + end + end + + VulnerabilitiesFinding.where(id: ids_to_delete).delete_all if ids_to_delete.any? + end +end diff --git a/lib/gitlab/background_migration/user_mentions/models/namespace.rb b/lib/gitlab/background_migration/user_mentions/models/namespace.rb index 6d7b9a86e69..8fa0db5fd4b 100644 --- a/lib/gitlab/background_migration/user_mentions/models/namespace.rb +++ b/lib/gitlab/background_migration/user_mentions/models/namespace.rb @@ -6,6 +6,7 @@ module Gitlab module Models # isolated Namespace model class Namespace < ApplicationRecord + include FeatureGate include ::Gitlab::VisibilityLevel include ::Gitlab::Utils::StrongMemoize include Gitlab::BackgroundMigration::UserMentions::Models::Concerns::Namespace::RecursiveTraversal diff --git a/lib/gitlab/changelog/ast.rb b/lib/gitlab/changelog/ast.rb new file mode 100644 index 00000000000..2c787d396f5 --- /dev/null +++ b/lib/gitlab/changelog/ast.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # AST nodes to evaluate when rendering a template. + # + # Evaluating an AST is done by walking over the nodes and calling + # `evaluate`. This method takes two arguments: + # + # 1. An instance of `EvalState`, used for tracking data such as the number + # of nested loops. + # 2. An object used as the data for the current scope. This can be an Array, + # Hash, String, or something else. It's up to the AST node to determine + # what to do with it. + # + # While tree walking interpreters (such as implemented here) aren't usually + # the fastest type of interpreter, they are: + # + # 1. Fast enough for our use case + # 2. Easy to implement and maintain + # + # In addition, our AST interpreter doesn't allow for arbitrary code + # execution, unlike existing template engines such as Mustache + # (https://github.com/mustache/mustache/issues/244) or ERB. + # + # Our interpreter also takes care of limiting the number of nested loops. + # And unlike Liquid, our interpreter is much smaller and thus has a smaller + # attack surface. Liquid isn't without its share of issues, such as + # https://github.com/Shopify/liquid/pull/1071. + # + # We also evaluated using Handlebars using the project + # https://github.com/SmartBear/ruby-handlebars. Sadly, this implementation + # of Handlebars doesn't support control of whitespace + # (https://github.com/SmartBear/ruby-handlebars/issues/37), and the project + # didn't appear to be maintained that much. + # + # This doesn't mean these template engines aren't good, instead it means + # they won't work for our use case. For more information, refer to the + # comment https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50063#note_469293322. + module AST + # An identifier in a selector. + Identifier = Struct.new(:name) do + def evaluate(state, data) + return data if name == 'it' + + data[name] if data.is_a?(Hash) + end + end + + # An integer used in a selector. + Integer = Struct.new(:value) do + def evaluate(state, data) + data[value] if data.is_a?(Array) + end + end + + # A selector used for loading a value. + Selector = Struct.new(:steps) do + def evaluate(state, data) + steps.reduce(data) do |current, step| + break if current.nil? + + step.evaluate(state, current) + end + end + end + + # A tag used for displaying a value in the output. + Variable = Struct.new(:selector) do + def evaluate(state, data) + selector.evaluate(state, data).to_s + end + end + + # A collection of zero or more expressions. + Expressions = Struct.new(:nodes) do + def evaluate(state, data) + nodes.map { |node| node.evaluate(state, data) }.join('') + end + end + + # A single text node. + Text = Struct.new(:text) do + def evaluate(*) + text + end + end + + # An `if` expression, with an optional `else` clause. + If = Struct.new(:condition, :true_body, :false_body) do + def evaluate(state, data) + result = + if truthy?(condition.evaluate(state, data)) + true_body.evaluate(state, data) + elsif false_body + false_body.evaluate(state, data) + end + + result.to_s + end + + def truthy?(value) + # We treat empty collections and such as false, removing the need for + # some sort of `if length(x) > 0` expression. + value.respond_to?(:empty?) ? !value.empty? : !!value + end + end + + # An `each` expression. + Each = Struct.new(:collection, :body) do + def evaluate(state, data) + values = collection.evaluate(state, data) + + return '' unless values.respond_to?(:each) + + # While unlikely to happen, it's possible users attempt to nest many + # loops in order to negatively impact the GitLab instance. To make + # this more difficult, we limit the number of nested loops a user can + # create. + state.enter_loop do + values.map { |value| body.evaluate(state, value) }.join('') + end + end + end + + # A class for transforming a raw Parslet AST into a more structured/easier + # to work with AST. + # + # For more information about Parslet transformations, refer to the + # documentation at http://kschiess.github.io/parslet/transform.html. + class Transformer < Parslet::Transform + rule(ident: simple(:name)) { Identifier.new(name.to_s) } + rule(int: simple(:name)) { Integer.new(name.to_i) } + rule(text: simple(:text)) { Text.new(text.to_s) } + rule(exprs: subtree(:nodes)) { Expressions.new(nodes) } + rule(selector: sequence(:steps)) { Selector.new(steps) } + rule(selector: simple(:step)) { Selector.new([step]) } + rule(variable: simple(:selector)) { Variable.new(selector) } + rule(each: simple(:values), body: simple(:body)) do + Each.new(values, body) + end + + rule(if: simple(:cond), true_body: simple(:true_body)) do + If.new(cond, true_body) + end + + rule( + if: simple(:cond), + true_body: simple(:true_body), + false_body: simple(:false_body) + ) do + If.new(cond, true_body, false_body) + end + end + end + end +end diff --git a/lib/gitlab/changelog/committer.rb b/lib/gitlab/changelog/committer.rb new file mode 100644 index 00000000000..31661650eff --- /dev/null +++ b/lib/gitlab/changelog/committer.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A class used for committing a release's changelog to a Git repository. + class Committer + def initialize(project, user) + @project = project + @user = user + end + + # Commits a release's changelog to a file on a branch. + # + # The `release` argument is a `Gitlab::Changelog::Release` for which to + # update the changelog. + # + # The `file` argument specifies the path to commit the changes to. + # + # The `branch` argument specifies the branch to commit the changes on. + # + # The `message` argument specifies the commit message to use. + def commit(release:, file:, branch:, message:) + # When retrying, we need to reprocess the existing changelog from + # scratch, otherwise we may end up throwing away changes. As such, all + # the logic is contained within the retry block. + Retriable.retriable(on: Error) do + commit = Gitlab::Git::Commit.last_for_path( + @project.repository, + branch, + file, + literal_pathspec: true + ) + + content = blob_content(file, commit) + + # If the release has already been added (e.g. concurrently by another + # API call), we don't want to add it again. + break if content&.match?(release.header_start_pattern) + + service = Files::MultiService.new( + @project, + @user, + commit_message: message, + branch_name: branch, + start_branch: branch, + actions: [ + { + action: content ? 'update' : 'create', + content: Generator.new(content.to_s).add(release), + file_path: file, + last_commit_id: commit&.sha + } + ] + ) + + result = service.execute + + raise Error.new(result[:message]) if result[:status] != :success + end + end + + def blob_content(file, commit = nil) + return unless commit + + @project.repository.blob_at(commit.sha, file)&.data + end + end + end +end diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb new file mode 100644 index 00000000000..105050936ce --- /dev/null +++ b/lib/gitlab/changelog/config.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # Configuration settings used when generating changelogs. + class Config + # When rendering changelog entries, authors are not included. + AUTHORS_NONE = 'none' + + # The path to the configuration file as stored in the project's Git + # repository. + FILE_PATH = '.gitlab/changelog_config.yml' + + # The default date format to use for formatting release dates. + DEFAULT_DATE_FORMAT = '%Y-%m-%d' + + # The default template to use for generating release sections. + DEFAULT_TEMPLATE = File.read(File.join(__dir__, 'template.tpl')) + + attr_accessor :date_format, :categories, :template + + def self.from_git(project) + if (yaml = project.repository.changelog_config) + from_hash(project, YAML.safe_load(yaml)) + else + new(project) + end + end + + def self.from_hash(project, hash) + config = new(project) + + if (date = hash['date_format']) + config.date_format = date + end + + if (template = hash['template']) + config.template = Parser.new.parse_and_transform(template) + end + + if (categories = hash['categories']) + if categories.is_a?(Hash) + config.categories = categories + else + raise Error, 'The "categories" configuration key must be a Hash' + end + end + + config + end + + def initialize(project) + @project = project + @date_format = DEFAULT_DATE_FORMAT + @template = Parser.new.parse_and_transform(DEFAULT_TEMPLATE) + @categories = {} + end + + def contributor?(user) + @project.team.contributor?(user) + end + + def category(name) + @categories[name] || name + end + + def format_date(date) + date.strftime(@date_format) + end + end + end +end diff --git a/lib/gitlab/changelog/error.rb b/lib/gitlab/changelog/error.rb new file mode 100644 index 00000000000..0bd886fbdb7 --- /dev/null +++ b/lib/gitlab/changelog/error.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # An error raised when a changelog couldn't be generated. + Error = Class.new(StandardError) + end +end diff --git a/lib/gitlab/changelog/eval_state.rb b/lib/gitlab/changelog/eval_state.rb new file mode 100644 index 00000000000..a0439df60cf --- /dev/null +++ b/lib/gitlab/changelog/eval_state.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A class for tracking state when evaluating a template + class EvalState + MAX_LOOPS = 4 + + def initialize + @loops = 0 + end + + def enter_loop + if @loops == MAX_LOOPS + raise Error, "You can only nest up to #{MAX_LOOPS} loops" + end + + @loops += 1 + retval = yield + @loops -= 1 + + retval + end + end + end +end diff --git a/lib/gitlab/changelog/generator.rb b/lib/gitlab/changelog/generator.rb new file mode 100644 index 00000000000..a80ca0728f9 --- /dev/null +++ b/lib/gitlab/changelog/generator.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # Parsing and generating of Markdown changelogs. + class Generator + # The regex used to parse a release header. + RELEASE_REGEX = + /^##\s+(?<version>#{Gitlab::Regex.unbounded_semver_regex})/.freeze + + # The `input` argument must be a `String` containing the existing + # changelog Markdown. If no changelog exists, this should be an empty + # `String`. + def initialize(input = '') + @lines = input.lines + @locations = {} + + @lines.each_with_index do |line, index| + matches = line.match(RELEASE_REGEX) + + next if !matches || !matches[:version] + + @locations[matches[:version]] = index + end + end + + # Generates the Markdown for the given release and returns the new + # changelog Markdown content. + # + # The `release` argument must be an instance of + # `Gitlab::Changelog::Release`. + def add(release) + versions = [release.version, *@locations.keys] + + VersionSorter.rsort!(versions) + + new_index = versions.index(release.version) + new_lines = @lines.dup + markdown = release.to_markdown + + if (insert_after = versions[new_index + 1]) + line_index = @locations[insert_after] + + new_lines.insert(line_index, markdown) + else + # When adding to the end of the changelog, the previous section only + # has a single newline, resulting in the release section title + # following it immediately. When this is the case, we insert an extra + # empty line to keep the changelog readable in its raw form. + new_lines.push("\n") if versions.length > 1 + new_lines.push(markdown.rstrip) + new_lines.push("\n") + end + + new_lines.join + end + end + end +end diff --git a/lib/gitlab/changelog/parser.rb b/lib/gitlab/changelog/parser.rb new file mode 100644 index 00000000000..a4c8da283cd --- /dev/null +++ b/lib/gitlab/changelog/parser.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A parser for the template syntax used for generating changelogs. + # + # As a quick primer on the template syntax, a basic template looks like + # this: + # + # {% each users %} + # Name: {{name}} + # Age: {{age}} + # + # {% if birthday %} + # This user is celebrating their birthday today! Yay! + # {% end %} + # {% end %} + # + # For more information, refer to the Parslet documentation found at + # http://kschiess.github.io/parslet/. + class Parser < Parslet::Parser + root(:exprs) + + rule(:exprs) do + ( + variable | if_expr | each_expr | escaped | text | newline + ).repeat.as(:exprs) + end + + rule(:space) { match('[ \\t]') } + rule(:whitespace) { match('\s').repeat } + rule(:lf) { str("\n") } + rule(:newline) { lf.as(:text) } + + # Escaped newlines are ignored, allowing the user to control the + # whitespace in the output. All other escape sequences are treated as + # literal text. + # + # For example, this: + # + # foo \ + # bar + # + # Is parsed into this: + # + # foo bar + rule(:escaped) do + backslash = str('\\') + + (backslash >> lf).ignore | (backslash >> chars).as(:text) + end + + # A sequence of regular characters, with the exception of newlines and + # escaped newlines. + rule(:chars) do + char = match("[^{\\\\\n]") + + # The rules here are such that we do treat single curly braces or + # non-opening tags (e.g. `{foo}`) as text, but not opening tags + # themselves (e.g. `{{`). + ( + char.repeat(1) | curly_open >> (curly_open | percent).absent? + ).repeat(1) + end + + rule(:text) { chars.as(:text) } + + # An integer, limited to 10 digits (= a 32 bits integer). + # + # The size is limited to prevents users from creating integers that are + # too large, as this may result in runtime errors. + rule(:integer) { match('\d').repeat(1, 10).as(:int) } + + # An identifier to look up in a data structure. + # + # We only support simple ASCII identifiers as we simply don't have a need + # for more complex identifiers (e.g. those containing multibyte + # characters). + rule(:ident) { match('[a-zA-Z_]').repeat(1).as(:ident) } + + # A selector is used for reading a value, consisting of one or more + # "steps". + # + # Examples: + # + # name + # users.0.name + # 0 + # it + rule(:selector) do + step = ident | integer + + whitespace >> + (step >> (str('.') >> step).repeat).as(:selector) >> + whitespace + end + + rule(:curly_open) { str('{') } + rule(:curly_close) { str('}') } + rule(:percent) { str('%') } + + # A variable tag. + # + # Examples: + # + # {{name}} + # {{users.0.name}} + rule(:variable) do + curly_open.repeat(2) >> selector.as(:variable) >> curly_close.repeat(2) + end + + rule(:expr_open) { curly_open >> percent >> whitespace } + rule(:expr_close) do + # Since whitespace control is important (as Markdown is whitespace + # sensitive), we default to stripping a newline that follows a %} tag. + # This is less annoying compared to having to opt-in to this behaviour. + whitespace >> percent >> curly_close >> lf.maybe.ignore + end + + rule(:end_tag) { expr_open >> str('end') >> expr_close } + + # An `if` expression, with an optional `else` clause. + # + # Examples: + # + # {% if foo %} + # yes + # {% end %} + # + # {% if foo %} + # yes + # {% else %} + # no + # {% end %} + rule(:if_expr) do + else_tag = + expr_open >> str('else') >> expr_close >> exprs.as(:false_body) + + expr_open >> + str('if') >> + space.repeat(1) >> + selector.as(:if) >> + expr_close >> + exprs.as(:true_body) >> + else_tag.maybe >> + end_tag + end + + # An `each` expression, used for iterating over collections. + # + # Example: + # + # {% each users %} + # * {{name}} + # {% end %} + rule(:each_expr) do + expr_open >> + str('each') >> + space.repeat(1) >> + selector.as(:each) >> + expr_close >> + exprs.as(:body) >> + end_tag + end + + def parse_and_transform(input) + AST::Transformer.new.apply(parse(input)) + rescue Parslet::ParseFailed => ex + # We raise a custom error so it's easier to catch different changelog + # related errors. In addition, this ensures the caller of this method + # doesn't depend on a Parslet specific error class. + raise Error.new("Failed to parse the template: #{ex.message}") + end + end + end +end diff --git a/lib/gitlab/changelog/release.rb b/lib/gitlab/changelog/release.rb new file mode 100644 index 00000000000..f2a01c2b0dc --- /dev/null +++ b/lib/gitlab/changelog/release.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A release to add to a changelog. + class Release + attr_reader :version + + def initialize(version:, date:, config:) + @version = version + @date = date + @config = config + @entries = Hash.new { |h, k| h[k] = [] } + + # This ensures that entries are presented in the same order as the + # categories Hash in the user's configuration. + @config.categories.values.each do |category| + @entries[category] = [] + end + end + + def add_entry( + title:, + commit:, + category:, + author: nil, + merge_request: nil + ) + # When changing these fields, keep in mind that this needs to be + # backwards compatible. For example, you can't just remove a field as + # this will break the changelog generation process for existing users. + entry = { + 'title' => title, + 'commit' => { + 'reference' => commit.to_reference(full: true), + 'trailers' => commit.trailers + } + } + + if author + entry['author'] = { + 'reference' => author.to_reference(full: true), + 'contributor' => @config.contributor?(author) + } + end + + if merge_request + entry['merge_request'] = { + 'reference' => merge_request.to_reference(full: true) + } + end + + @entries[@config.category(category)] << entry + end + + def to_markdown + state = EvalState.new + data = { 'categories' => entries_for_template } + + # While not critical, we would like release sections to be separated by + # an empty line in the changelog; ensuring it's readable even in its + # raw form. + # + # Since it can be a bit tricky to get this right in a template, we + # enforce an empty line separator ourselves. + markdown = @config.template.evaluate(state, data).strip + + # The release header can't be changed using the Liquid template, as we + # need this to be in a known format. Without this restriction, we won't + # know where to insert a new release section in an existing changelog. + "## #{@version} (#{release_date})\n\n#{markdown}\n\n" + end + + def header_start_pattern + /^##\s*#{Regexp.escape(@version)}/ + end + + private + + def release_date + @config.format_date(@date) + end + + def entries_for_template + rows = [] + + @entries.each do |category, entries| + next if entries.empty? + + rows << { + 'title' => category, + 'count' => entries.length, + 'single_change' => entries.length == 1, + 'entries' => entries + } + end + + rows + end + end + end +end diff --git a/lib/gitlab/changelog/template.tpl b/lib/gitlab/changelog/template.tpl new file mode 100644 index 00000000000..584939dff51 --- /dev/null +++ b/lib/gitlab/changelog/template.tpl @@ -0,0 +1,15 @@ +{% if categories %} +{% each categories %} +### {{ title }} ({% if single_change %}1 change{% else %}{{ count }} changes{% end %}) + +{% each entries %} +- [{{ title }}]({{ commit.reference }})\ +{% if author.contributor %} by {{ author.reference }}{% end %}\ +{% if merge_request %} ([merge request]({{ merge_request.reference }})){% end %} + +{% end %} + +{% end %} +{% else %} +No changes. +{% end %} diff --git a/lib/gitlab/chaos.rb b/lib/gitlab/chaos.rb index 911f2993b8a..029a9210dc9 100644 --- a/lib/gitlab/chaos.rb +++ b/lib/gitlab/chaos.rb @@ -47,5 +47,13 @@ module Gitlab def self.kill Process.kill("KILL", Process.pid) end + + def self.run_gc + # Tenure any live objects from young-gen to old-gen + 4.times { GC.start(full_mark: false) } + # Run a full mark-and-sweep collection + GC.start + GC.stat + end end end diff --git a/lib/gitlab/badge/base.rb b/lib/gitlab/ci/badge/base.rb index fb55b9e2f1f..c65f120753d 100644 --- a/lib/gitlab/badge/base.rb +++ b/lib/gitlab/ci/badge/base.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge class Base def entity diff --git a/lib/gitlab/badge/coverage/metadata.rb b/lib/gitlab/ci/badge/coverage/metadata.rb index 9181ba2d4b0..7654b6d6fc5 100644 --- a/lib/gitlab/badge/coverage/metadata.rb +++ b/lib/gitlab/ci/badge/coverage/metadata.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Coverage ## diff --git a/lib/gitlab/badge/coverage/report.rb b/lib/gitlab/ci/badge/coverage/report.rb index 390da014a5a..28863a0703b 100644 --- a/lib/gitlab/badge/coverage/report.rb +++ b/lib/gitlab/ci/badge/coverage/report.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Coverage ## diff --git a/lib/gitlab/badge/coverage/template.rb b/lib/gitlab/ci/badge/coverage/template.rb index 1b985f83b22..7589fa5ff8b 100644 --- a/lib/gitlab/badge/coverage/template.rb +++ b/lib/gitlab/ci/badge/coverage/template.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Coverage ## diff --git a/lib/gitlab/badge/metadata.rb b/lib/gitlab/ci/badge/metadata.rb index b9ae68134b0..eec9fedfaa9 100644 --- a/lib/gitlab/badge/metadata.rb +++ b/lib/gitlab/ci/badge/metadata.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge ## # Abstract class for badge metadata diff --git a/lib/gitlab/badge/pipeline/metadata.rb b/lib/gitlab/ci/badge/pipeline/metadata.rb index d4d789558c9..2aa08476336 100644 --- a/lib/gitlab/badge/pipeline/metadata.rb +++ b/lib/gitlab/ci/badge/pipeline/metadata.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Pipeline ## diff --git a/lib/gitlab/badge/pipeline/status.rb b/lib/gitlab/ci/badge/pipeline/status.rb index f061ba22688..a2ee2642872 100644 --- a/lib/gitlab/badge/pipeline/status.rb +++ b/lib/gitlab/ci/badge/pipeline/status.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Pipeline ## diff --git a/lib/gitlab/badge/pipeline/template.rb b/lib/gitlab/ci/badge/pipeline/template.rb index af8e318395b..8430b01fc9a 100644 --- a/lib/gitlab/badge/pipeline/template.rb +++ b/lib/gitlab/ci/badge/pipeline/template.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge module Pipeline ## diff --git a/lib/gitlab/badge/template.rb b/lib/gitlab/ci/badge/template.rb index 9ac8f1c17f2..0580dad72ba 100644 --- a/lib/gitlab/badge/template.rb +++ b/lib/gitlab/ci/badge/template.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gitlab +module Gitlab::Ci module Badge ## # Abstract template class for badges diff --git a/lib/gitlab/ci/build/credentials/base.rb b/lib/gitlab/ci/build/credentials/base.rb index 58adf6e506d..2aeb8453703 100644 --- a/lib/gitlab/ci/build/credentials/base.rb +++ b/lib/gitlab/ci/build/credentials/base.rb @@ -6,7 +6,7 @@ module Gitlab module Credentials class Base def type - self.class.name.demodulize.underscore + raise NotImplementedError end end end diff --git a/lib/gitlab/ci/build/credentials/factory.rb b/lib/gitlab/ci/build/credentials/factory.rb index fa805abb8bb..e8996cb9dc4 100644 --- a/lib/gitlab/ci/build/credentials/factory.rb +++ b/lib/gitlab/ci/build/credentials/factory.rb @@ -20,7 +20,7 @@ module Gitlab end def providers - [Registry] + [Registry::GitlabRegistry, Registry::DependencyProxy] end end end diff --git a/lib/gitlab/ci/build/credentials/registry.rb b/lib/gitlab/ci/build/credentials/registry.rb deleted file mode 100644 index 1c8588d9913..00000000000 --- a/lib/gitlab/ci/build/credentials/registry.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Build - module Credentials - class Registry < Base - attr_reader :username, :password - - def initialize(build) - @username = 'gitlab-ci-token' - @password = build.token - end - - def url - Gitlab.config.registry.host_port - end - - def valid? - Gitlab.config.registry.enabled - end - end - end - end - end -end diff --git a/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb new file mode 100644 index 00000000000..b6ac06cfb53 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/registry/dependency_proxy.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Credentials + module Registry + class DependencyProxy < GitlabRegistry + def url + "#{Gitlab.config.gitlab.host}:#{Gitlab.config.gitlab.port}" + end + + def valid? + Gitlab.config.dependency_proxy.enabled + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/credentials/registry/gitlab_registry.rb b/lib/gitlab/ci/build/credentials/registry/gitlab_registry.rb new file mode 100644 index 00000000000..5bd30e677e9 --- /dev/null +++ b/lib/gitlab/ci/build/credentials/registry/gitlab_registry.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Build + module Credentials + module Registry + class GitlabRegistry < Credentials::Base + attr_reader :username, :password + + def initialize(build) + @username = Gitlab::Auth::CI_JOB_USER + @password = build.token + end + + def url + Gitlab.config.registry.host_port + end + + def valid? + Gitlab.config.registry.enabled + end + + def type + 'registry' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index a39afee194c..2d4f9cf635b 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -7,30 +7,17 @@ module Gitlab include ::Gitlab::Utils::StrongMemoize Result = Struct.new(:when, :start_in, :allow_failure, :variables) do - def build_attributes(seed_attributes = {}) + def build_attributes { when: self.when, options: { start_in: start_in }.compact, - allow_failure: allow_failure, - yaml_variables: yaml_variables(seed_attributes[:yaml_variables]) + allow_failure: allow_failure }.compact end def pass? self.when != 'never' end - - private - - def yaml_variables(seed_variables) - return unless variables && seed_variables - - indexed_seed_variables = seed_variables.deep_dup.index_by { |var| var[:key] } - - variables.each_with_object(indexed_seed_variables) do |var, hash| - hash[var[0].to_s] = { key: var[0].to_s, value: var[1], public: true } - end.values - end end def initialize(rule_hashes, default_when:) diff --git a/lib/gitlab/ci/charts.rb b/lib/gitlab/ci/charts.rb index 25fb9c0ca97..797193a6be5 100644 --- a/lib/gitlab/ci/charts.rb +++ b/lib/gitlab/ci/charts.rb @@ -31,9 +31,10 @@ module Gitlab current = @from while current <= @to - @labels << current.strftime(@format) - @total << (totals_count[current] || 0) - @success << (success_count[current] || 0) + label = current.strftime(@format) + @labels << label + @total << (totals_count[label] || 0) + @success << (success_count[label] || 0) current += interval_step end @@ -45,6 +46,7 @@ module Gitlab query .group("date_trunc('#{interval}', #{::Ci::Pipeline.table_name}.created_at)") .count(:created_at) + .transform_keys { |date| date.strftime(@format) } end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 8ed4dc61920..dbb48a81030 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -13,7 +13,8 @@ module Gitlab RESCUE_ERRORS = [ Gitlab::Config::Loader::FormatError, Extendable::ExtensionError, - External::Processor::IncludeError + External::Processor::IncludeError, + Config::Yaml::Tags::TagError ].freeze attr_reader :root @@ -89,9 +90,10 @@ module Gitlab end def build_config(config) - initial_config = Gitlab::Config::Loader::Yaml.new(config).load! + initial_config = Config::Yaml.load!(config) initial_config = Config::External::Processor.new(initial_config, @context).perform initial_config = Config::Extendable.new(initial_config).to_hash + initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash initial_config = Config::EdgeStagesInjector.new(initial_config).to_hash initial_config diff --git a/lib/gitlab/ci/config/entry/commands.rb b/lib/gitlab/ci/config/entry/commands.rb index 7a86fca3056..341f87b44ab 100644 --- a/lib/gitlab/ci/config/entry/commands.rb +++ b/lib/gitlab/ci/config/entry/commands.rb @@ -10,12 +10,14 @@ module Gitlab class Commands < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + MAX_NESTING_LEVEL = 10 + validations do - validates :config, string_or_nested_array_of_strings: true + validates :config, string_or_nested_array_of_strings: { max_level: MAX_NESTING_LEVEL } end def value - Array(@config).flatten(1) + Array(@config).flatten(MAX_NESTING_LEVEL) end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 85e3514499c..a20b802be58 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script type image services start_in artifacts cache dependencies before_script after_script environment coverage retry parallel interruptible timeout - resource_group release secrets].freeze + release secrets].freeze REQUIRED_BY_NEEDS = %i[stage].freeze @@ -30,7 +30,6 @@ module Gitlab } validates :dependencies, array_of_strings: true - validates :resource_group, type: String validates :allow_failure, hash_or_boolean: true end @@ -124,7 +123,7 @@ module Gitlab attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, - :interruptible, :timeout, :resource_group, + :interruptible, :timeout, :release, :allow_failure def self.matching?(name, config) @@ -174,7 +173,6 @@ module Gitlab ignore: ignored?, allow_failure_criteria: allow_failure_criteria, needs: needs_defined? ? needs_value : nil, - resource_group: resource_group, scheduling_type: needs_defined? ? :dag : :stage ).compact end @@ -186,8 +184,6 @@ module Gitlab private def allow_failure_criteria - return unless ::Gitlab::Ci::Features.allow_failure_with_exit_codes_enabled? - if allow_failure_defined? && allow_failure_value.is_a?(Hash) allow_failure_value end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 5ef8cfbddb7..9584d19bdec 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -15,7 +15,7 @@ module Gitlab include ::Gitlab::Config::Entry::Inheritable PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables - inherit allow_failure when needs].freeze + inherit allow_failure when needs resource_group].freeze included do validations do @@ -32,6 +32,7 @@ module Gitlab with_options allow_nil: true do validates :extends, array_of_strings_or_string: true validates :rules, array_of_hashes: true + validates :resource_group, type: String end end @@ -64,7 +65,7 @@ module Gitlab inherit: false, default: {} - attributes :extends, :rules + attributes :extends, :rules, :resource_group end def compose!(deps = nil) @@ -125,7 +126,8 @@ module Gitlab rules: rules_value, variables: root_and_job_variables_value, only: only_value, - except: except_value }.compact + except: except_value, + resource_group: resource_group }.compact end def root_and_job_variables_value diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 4684a9eb981..7d3fddd850d 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -60,7 +60,7 @@ module Gitlab def content_hash strong_memoize(:content_yaml) do - Gitlab::Config::Loader::Yaml.new(content).load! + ::Gitlab::Ci::Config::Yaml.load!(content) end rescue Gitlab::Config::Loader::FormatError nil diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 4d91cfd4c57..b85b7a9edeb 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -99,8 +99,6 @@ module Gitlab end def expand_variables(data) - return data unless ::Feature.enabled?(:variables_in_include_section_ci) - if data.is_a?(String) expand(data) else diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb new file mode 100644 index 00000000000..de833619c8d --- /dev/null +++ b/lib/gitlab/ci/config/yaml.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze + + class << self + def load!(content) + ensure_custom_tags + + Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + end + + private + + def ensure_custom_tags + @ensure_custom_tags ||= begin + AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } + + true + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/tags.rb b/lib/gitlab/ci/config/yaml/tags.rb new file mode 100644 index 00000000000..1575edad3b0 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/tags.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + module Tags + TagError = Class.new(StandardError) + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/tags/base.rb b/lib/gitlab/ci/config/yaml/tags/base.rb new file mode 100644 index 00000000000..13416a4afb6 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/tags/base.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + module Tags + class Base + CircularReferenceError = Class.new(Tags::TagError) + NotValidError = Class.new(Tags::TagError) + + extend ::Gitlab::Utils::Override + + attr_accessor :resolved_status, :resolved_value, :data + + def self.tag + raise NotImplementedError + end + + # Only one of the `seq`, `scalar`, `map` fields is available. + def init_with(coder) + @data = { + tag: coder.tag, # This is the custom YAML tag, like !reference or !flatten + style: coder.style, + seq: coder.seq, # This holds Array data + scalar: coder.scalar, # This holds data of basic types, like String. + map: coder.map # This holds Hash data. + } + end + + def valid? + raise NotImplementedError + end + + def resolve(resolver) + raise NotValidError, validation_error_message unless valid? + raise CircularReferenceError, circular_error_message if resolving? + return resolved_value if resolved? + + self.resolved_status = :in_progress + self.resolved_value = _resolve(resolver) + self.resolved_status = :done + resolved_value + end + + private + + def _resolve(resolver) + raise NotImplementedError + end + + def resolved? + resolved_status == :done + end + + def resolving? + resolved_status == :in_progress + end + + def circular_error_message + "#{data[:tag]} #{data[:seq].inspect} is part of a circular chain" + end + + def validation_error_message + "#{data[:tag]} #{(data[:scalar].presence || data[:map].presence || data[:seq]).inspect} is not valid" + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/tags/reference.rb b/lib/gitlab/ci/config/yaml/tags/reference.rb new file mode 100644 index 00000000000..22822614b67 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/tags/reference.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + module Tags + class Reference < Base + MissingReferenceError = Class.new(Tags::TagError) + + def self.tag + '!reference' + end + + override :valid? + def valid? + data[:seq].is_a?(Array) && + !data[:seq].empty? && + data[:seq].all? { |identifier| identifier.is_a?(String) } + end + + private + + def location + data[:seq].to_a.map(&:to_sym) + end + + override :_resolve + def _resolve(resolver) + object = resolver.config.dig(*location) + value = resolver.deep_resolve(object) + + raise MissingReferenceError, missing_ref_error_message unless value + + value + end + + def missing_ref_error_message + "#{data[:tag]} #{data[:seq].inspect} could not be found" + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml/tags/resolver.rb b/lib/gitlab/ci/config/yaml/tags/resolver.rb new file mode 100644 index 00000000000..e207ec296b6 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/tags/resolver.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + module Tags + # This class is the entry point for transforming custom YAML tags back + # into primitive objects. + # Usage: `Resolver.new(a_hash_including_custom_tag_objects).to_hash` + # + class Resolver + attr_reader :config + + def initialize(config) + @config = config.deep_dup + end + + def to_hash + deep_resolve(config) + end + + def deep_resolve(object) + case object + when Array + object.map(&method(:resolve_wrapper)) + when Hash + object.deep_transform_values(&method(:resolve_wrapper)) + else + resolve_wrapper(object) + end + end + + def resolve_wrapper(object) + if object.respond_to?(:resolve) + object.resolve(self) + else + object + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index 7956cf14203..d1a366125ef 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -55,21 +55,30 @@ module Gitlab ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false) end - def self.pipeline_open_merge_requests?(project) - ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: true) - end - def self.ci_pipeline_editor_page_enabled?(project) ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: :yaml) end - def self.allow_failure_with_exit_codes_enabled? - ::Feature.enabled?(:ci_allow_failure_with_exit_codes, default_enabled: :yaml) - end - def self.rules_variables_enabled?(project) ::Feature.enabled?(:ci_rules_variables, project, default_enabled: true) end + + def self.validate_build_dependencies?(project) + ::Feature.enabled?(:ci_validate_build_dependencies, project, default_enabled: :yaml) && + ::Feature.disabled?(:ci_validate_build_dependencies_override, project) + end + + def self.display_quality_on_mr_diff?(project) + ::Feature.enabled?(:codequality_mr_diff, project, default_enabled: false) + end + + def self.display_codequality_backend_comparison?(project) + ::Feature.enabled?(:codequality_backend_comparison, project, default_enabled: :yaml) + end + + def self.use_coverage_data_new_finder?(record) + ::Feature.enabled?(:coverage_data_new_finder, record, default_enabled: :yaml) + end end end end diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index a8943eadf4f..0870c74053a 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -45,7 +45,7 @@ module Gitlab end def custom_claims - { + fields = { namespace_id: namespace.id.to_s, namespace_path: namespace.full_path, project_id: project.id.to_s, @@ -59,6 +59,15 @@ module Gitlab ref_type: ref_type, ref_protected: build.protected.to_s } + + if include_environment_claims? + fields.merge!( + environment: environment.name, + environment_protected: environment_protected?.to_s + ) + end + + fields end def key @@ -102,6 +111,20 @@ module Gitlab def ref_type ::Ci::BuildRunnerPresenter.new(build).ref_type end + + def environment + build.persisted_environment + end + + def environment_protected? + false # Overridden in EE + end + + def include_environment_claims? + Feature.enabled?(:ci_jwt_include_environment) && environment.present? + end end end end + +Gitlab::Ci::Jwt.prepend_if_ee('::EE::Gitlab::Ci::Jwt') diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 985639982aa..2baa8faf849 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -20,6 +20,10 @@ module Gitlab rescue KeyError raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" end + + def self.instrument! + parsers.values.each { |parser_class| parser_class.prepend(Parsers::Instrumentation) } + end end end end diff --git a/lib/gitlab/ci/parsers/instrumentation.rb b/lib/gitlab/ci/parsers/instrumentation.rb new file mode 100644 index 00000000000..ab4a923d9aa --- /dev/null +++ b/lib/gitlab/ci/parsers/instrumentation.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Instrumentation + BUCKETS = [0.25, 1, 5, 10].freeze + + def parse!(*args) + parser_result = nil + + duration = Benchmark.realtime do + parser_result = super + end + + labels = {} + + histogram = Gitlab::Metrics.histogram( + :ci_report_parser_duration_seconds, + 'Duration of parsing a CI report artifact', + labels, + BUCKETS + ) + + histogram.observe({ parser: self.class.name }, duration) + + parser_result + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index f0548284001..d3bc3a38f1f 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -23,7 +23,7 @@ module Gitlab pipeline_schedule: @command.schedule, merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, - locked: @command.project.latest_pipeline_locked, + locked: @command.project.default_pipeline_lock, variables_attributes: variables_attributes ) end diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb index 2ca51930c19..f0214bb4e38 100644 --- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -25,7 +25,7 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def auto_cancelable_pipelines - pipelines + project.all_pipelines.ci_and_parent_sources .where(ref: pipeline.ref) .where.not(id: pipeline.same_family_pipeline_ids) .where.not(sha: project.commit(pipeline.ref).try(:id)) @@ -33,14 +33,6 @@ module Gitlab .with_only_interruptible_builds end # rubocop: enable CodeReuse/ActiveRecord - - def pipelines - if ::Feature.enabled?(:ci_auto_cancel_all_pipelines, project, default_enabled: true) - project.all_pipelines.ci_and_parent_sources - else - project.ci_pipelines - end - end end end end diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index 5314fd471c3..a7680f6e593 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -34,16 +34,22 @@ module Gitlab private def find_config - SOURCES.each do |source| + sources.each do |source| config = source.new(@pipeline, @command) return config if config.exists? end nil end + + def sources + SOURCES + end end end end end end end + +Gitlab::Ci::Pipeline::Chain::Config::Content.prepend_if_ee('EE::Gitlab::Ci::Pipeline::Chain::Config::Content') diff --git a/lib/gitlab/ci/pipeline/chain/template_usage.rb b/lib/gitlab/ci/pipeline/chain/template_usage.rb index c1a7b4ed453..2fcf1740b5f 100644 --- a/lib/gitlab/ci/pipeline/chain/template_usage.rb +++ b/lib/gitlab/ci/pipeline/chain/template_usage.rb @@ -19,7 +19,7 @@ module Gitlab def track_event(template) Gitlab::UsageDataCounters::CiTemplateUniqueCounter - .track_unique_project_event(project_id: pipeline.project_id, template: template) + .track_unique_project_event(project_id: pipeline.project_id, template: template, config_source: pipeline.config_source) end def included_templates diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index db6cca27f1c..c77f4dcca5a 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -45,6 +45,15 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end end + + def legacy_update_jobs_counter + strong_memoize(:legacy_update_jobs_counter) do + name = :ci_legacy_update_jobs_as_retried_total + comment = 'Counter of occurrences when jobs were not being set as retried before update_retried' + + Gitlab::Metrics.counter(name, comment) + end + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index fe3c2bca551..3770bb4b328 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -73,17 +73,28 @@ module Gitlab def to_resource strong_memoize(:resource) do - if bridge? - ::Ci::Bridge.new(attributes) - else - ::Ci::Build.new(attributes).tap do |build| - build.assign_attributes(self.class.environment_attributes_for(build)) - build.resource_group = Seed::Build::ResourceGroup.new(build, @resource_group_key).to_resource - end + processable = initialize_processable + assign_resource_group(processable) + processable + end + end + + def initialize_processable + if bridge? + ::Ci::Bridge.new(attributes) + else + ::Ci::Build.new(attributes).tap do |build| + build.assign_attributes(self.class.environment_attributes_for(build)) end end end + def assign_resource_group(processable) + processable.resource_group = + Seed::Processable::ResourceGroup.new(processable, @resource_group_key) + .to_resource + end + def self.environment_attributes_for(build) return {} unless build.has_environment? @@ -159,7 +170,11 @@ module Gitlab next {} unless @using_rules if ::Gitlab::Ci::Features.rules_variables_enabled?(@pipeline.project) - rules_result.build_attributes(@seed_attributes) + rules_variables_result = ::Gitlab::Ci::Variables::Helpers.merge_variables( + @seed_attributes[:yaml_variables], rules_result.variables + ) + + rules_result.build_attributes.merge(yaml_variables: rules_variables_result) else rules_result.build_attributes end @@ -188,7 +203,6 @@ module Gitlab # we need to prevent the exit codes from being persisted because they # would break the behavior defined by `rules:allow_failure`. def allow_failure_criteria_attributes - return {} unless ::Gitlab::Ci::Features.allow_failure_with_exit_codes_enabled? return {} if rules_attributes[:allow_failure].nil? return {} unless @seed_attributes.dig(:options, :allow_failure_criteria) diff --git a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb index c0641d9ff0a..f8ea6d4184c 100644 --- a/lib/gitlab/ci/pipeline/seed/build/resource_group.rb +++ b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb @@ -4,21 +4,21 @@ module Gitlab module Ci module Pipeline module Seed - class Build + module Processable class ResourceGroup < Seed::Base include Gitlab::Utils::StrongMemoize - attr_reader :build, :resource_group_key + attr_reader :processable, :resource_group_key - def initialize(build, resource_group_key) - @build = build + def initialize(processable, resource_group_key) + @processable = processable @resource_group_key = resource_group_key end def to_resource return unless resource_group_key.present? - resource_group = build.project.resource_groups + resource_group = processable.project.resource_groups .safe_find_or_create_by(key: expanded_resource_group_key) resource_group if resource_group.persisted? @@ -28,7 +28,7 @@ module Gitlab def expanded_resource_group_key strong_memoize(:expanded_resource_group_key) do - ExpandVariables.expand(resource_group_key, -> { build.simple_variables }) + ExpandVariables.expand(resource_group_key, -> { processable.simple_variables }) end end end diff --git a/lib/gitlab/ci/reports/codequality_mr_diff.rb b/lib/gitlab/ci/reports/codequality_mr_diff.rb new file mode 100644 index 00000000000..e60a075e3f5 --- /dev/null +++ b/lib/gitlab/ci/reports/codequality_mr_diff.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class CodequalityMrDiff + attr_reader :files + + def initialize(raw_report) + @raw_report = raw_report + @files = {} + build_report! + end + + private + + def build_report! + codequality_files = @raw_report.all_degradations.each_with_object({}) do |degradation, codequality_files| + unless codequality_files[degradation.dig(:location, :path)].present? + codequality_files[degradation.dig(:location, :path)] = [] + end + + build_mr_diff_payload(codequality_files, degradation) + end + + @files = codequality_files + end + + def build_mr_diff_payload(codequality_files, degradation) + codequality_files[degradation.dig(:location, :path)] << { + line: degradation.dig(:location, :lines, :begin) || degradation.dig(:location, :positions, :begin, :line), + description: degradation[:description], + severity: degradation[:severity] + } + end + end + end + end +end diff --git a/lib/gitlab/ci/status/bridge/factory.rb b/lib/gitlab/ci/status/bridge/factory.rb index b9bd66cee71..4d5a94a3beb 100644 --- a/lib/gitlab/ci/status/bridge/factory.rb +++ b/lib/gitlab/ci/status/bridge/factory.rb @@ -8,6 +8,7 @@ module Gitlab def self.extended_statuses [[Status::Bridge::Failed], [Status::Bridge::Manual], + [Status::Bridge::WaitingForResource], [Status::Bridge::Play], [Status::Bridge::Action]] end diff --git a/lib/gitlab/ci/status/bridge/waiting_for_resource.rb b/lib/gitlab/ci/status/bridge/waiting_for_resource.rb new file mode 100644 index 00000000000..d2c8f71a609 --- /dev/null +++ b/lib/gitlab/ci/status/bridge/waiting_for_resource.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Bridge + class WaitingForResource < Status::Processable::WaitingForResource + end + end + end + end +end diff --git a/lib/gitlab/ci/status/build/waiting_for_resource.rb b/lib/gitlab/ci/status/build/waiting_for_resource.rb index 008e6a17bdd..5dcc060a990 100644 --- a/lib/gitlab/ci/status/build/waiting_for_resource.rb +++ b/lib/gitlab/ci/status/build/waiting_for_resource.rb @@ -4,22 +4,7 @@ module Gitlab module Ci module Status module Build - class WaitingForResource < Status::Extended - ## - # TODO: image is shared with 'pending' - # until we get a dedicated one - # - def illustration - { - image: 'illustrations/pending_job_empty.svg', - size: 'svg-430', - title: _('This job is waiting for resource: ') + subject.resource_group.key - } - end - - def self.matches?(build, _) - build.waiting_for_resource? - end + class WaitingForResource < Status::Processable::WaitingForResource end end end diff --git a/lib/gitlab/ci/status/processable/waiting_for_resource.rb b/lib/gitlab/ci/status/processable/waiting_for_resource.rb new file mode 100644 index 00000000000..c9b1dd795d0 --- /dev/null +++ b/lib/gitlab/ci/status/processable/waiting_for_resource.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + module Processable + class WaitingForResource < Status::Extended + ## + # TODO: image is shared with 'pending' + # until we get a dedicated one + # + def illustration + { + image: 'illustrations/pending_job_empty.svg', + size: 'svg-430', + title: _('This job is waiting for resource: ') + subject.resource_group.key + } + end + + def self.matches?(processable, _) + processable.waiting_for_resource? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 501d8737acd..daed75a42ee 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.19" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.22" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 192b1509fdc..6f30fc2dcd5 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.37.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.40.0" environment: name: production variables: diff --git a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml index 84bb0ff3b33..8f64da24410 100644 --- a/lib/gitlab/ci/templates/Maven.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Maven.gitlab-ci.yml @@ -40,14 +40,14 @@ verify:jdk8: <<: *verify # To deploy packages from CI, create a ci_settings.xml file -# For deploying packages to GitLab's Maven Repository: See https://docs.gitlab.com/ee/user/project/packages/maven_repository.html#creating-maven-packages-with-gitlab-cicd for more details. +# For deploying packages to GitLab's Maven Repository: See https://docs.gitlab.com/ee/user/packages/maven_repository/index.html#create-maven-packages-with-gitlab-cicd for more details. # Please note: The GitLab Maven Repository is currently only available in GitLab Premium / Ultimate. # For `master` branch run `mvn deploy` automatically. deploy:jdk8: stage: deploy script: - if [ ! -f ci_settings.xml ]; - then echo "CI settings missing\! If deploying to GitLab Maven Repository, please see https://docs.gitlab.com/ee/user/project/packages/maven_repository.html#creating-maven-packages-with-gitlab-cicd for instructions."; + then echo "CI settings missing\! If deploying to GitLab Maven Repository, please see https://docs.gitlab.com/ee/user/packages/maven_repository/index.html#create-maven-packages-with-gitlab-cicd for instructions."; fi - 'mvn $MAVEN_CLI_OPTS deploy -s ci_settings.xml' only: diff --git a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml index 63237e41376..21e926ef275 100644 --- a/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/License-Scanning.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/license_compliance/ +# Read more about this feature here: https://docs.gitlab.com/ee/user/compliance/license_compliance/index.html # # Configure the scanning tool through the environment variables. # List of the variables: https://gitlab.com/gitlab-org/security-products/analyzers/license-finder#settings @@ -21,7 +21,6 @@ license_scanning: LM_REPORT_VERSION: '2.1' SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD allow_failure: true - needs: [] script: - /run.sh analyze . artifacts: diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index 56c6fbd96bc..828352743b4 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -9,7 +9,7 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" - SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf" + SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf, semgrep" SAST_EXCLUDED_ANALYZERS: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" SAST_ANALYZER_IMAGE_TAG: 2 @@ -66,7 +66,8 @@ brakeman-sast: - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /brakeman/ exists: - - 'config/routes.rb' + - '**/*.rb' + - '**/Gemfile' eslint-sast: extends: .sast-analyzer @@ -243,6 +244,23 @@ security-code-scan-sast: - '**/*.csproj' - '**/*.vbproj' +semgrep-sast: + extends: .sast-analyzer + image: + name: "$SAST_ANALYZER_IMAGE" + variables: + SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:latest" + rules: + - if: $SAST_DISABLED + when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ + when: never + - if: $CI_COMMIT_BRANCH && + $SAST_DEFAULT_ANALYZERS =~ /semgrep/ && + $SAST_EXPERIMENTAL_FEATURES == 'true' + exists: + - '**/*.py' + sobelow-sast: extends: .sast-analyzer image: diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 0222ca021b7..3258d965c93 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -182,7 +182,7 @@ module Gitlab if job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| archive_stream!(stream) - stream.destroy! + destroy_stream(job) { stream.destroy! } end elsif current_path File.open(current_path) do |stream| @@ -268,7 +268,21 @@ module Gitlab end def trace_artifact - job.job_artifacts_trace + read_trace_artifact(job) { job.job_artifacts_trace } + end + + ## + # Overridden in EE + # + def destroy_stream(job) + yield + end + + ## + # Overriden in EE + # + def read_trace_artifact(job) + yield end def being_watched_cache_key @@ -277,3 +291,5 @@ module Gitlab end end end + +::Gitlab::Ci::Trace.prepend_if_ee('EE::Gitlab::Ci::Trace') diff --git a/lib/gitlab/ci/trace/checksum.rb b/lib/gitlab/ci/trace/checksum.rb index 7cdb6a6c03c..92bed817875 100644 --- a/lib/gitlab/ci/trace/checksum.rb +++ b/lib/gitlab/ci/trace/checksum.rb @@ -30,7 +30,11 @@ module Gitlab end def state_crc32 - strong_memoize(:state_crc32) { build.pending_state&.crc32 } + strong_memoize(:state_crc32) do + ::Gitlab::Database::Consistency.with_read_consistency do + build.pending_state&.crc32 + end + end end def chunks_crc32 @@ -59,8 +63,10 @@ module Gitlab # def trace_chunks strong_memoize(:trace_chunks) do - build.trace_chunks.persisted - .select(::Ci::BuildTraceChunk.metadata_attributes) + ::Ci::BuildTraceChunk.with_read_consistency(build) do + build.trace_chunks.persisted + .select(::Ci::BuildTraceChunk.metadata_attributes) + end end end diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 6f3e4ccf48d..7c2e39b1e53 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -227,12 +227,20 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - def build_chunk - @chunks_cache[chunk_index] = ::Ci::BuildTraceChunk.new(build: build, chunk_index: chunk_index) + def next_chunk + @chunks_cache[chunk_index] = begin + if ::Ci::BuildTraceChunk.consistent_reads_enabled?(build) + ::Ci::BuildTraceChunk + .safe_find_or_create_by(build: build, chunk_index: chunk_index) + else + ::Ci::BuildTraceChunk + .new(build: build, chunk_index: chunk_index) + end + end end def ensure_chunk - current_chunk || build_chunk + current_chunk || next_chunk || current_chunk end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/variables/collection/sorted.rb b/lib/gitlab/ci/variables/collection/sorted.rb index 6abc6a5644f..e641df10462 100644 --- a/lib/gitlab/ci/variables/collection/sorted.rb +++ b/lib/gitlab/ci/variables/collection/sorted.rb @@ -8,8 +8,9 @@ module Gitlab include TSort include Gitlab::Utils::StrongMemoize - def initialize(variables) + def initialize(variables, project) @variables = variables + @project = project end def valid? @@ -19,7 +20,7 @@ module Gitlab # errors sorts an array of variables, ignoring unknown variable references, # and returning an error string if a circular variable reference is found def errors - return if Feature.disabled?(:variable_inside_variable) + return if Feature.disabled?(:variable_inside_variable, @project) strong_memoize(:errors) do # Check for cyclic dependencies and build error message in that case @@ -34,7 +35,7 @@ module Gitlab # sort sorts an array of variables, ignoring unknown variable references. # If a circular variable reference is found, the original array is returned def sort - return @variables if Feature.disabled?(:variable_inside_variable) + return @variables if Feature.disabled?(:variable_inside_variable, @project) return @variables if errors tsort diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb new file mode 100644 index 00000000000..e2a54f90ecb --- /dev/null +++ b/lib/gitlab/ci/variables/helpers.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + module Helpers + class << self + def merge_variables(current_vars, new_vars) + current_vars = transform_from_yaml_variables(current_vars) + new_vars = transform_from_yaml_variables(new_vars) + + transform_to_yaml_variables( + current_vars.merge(new_vars) + ) + end + + def transform_to_yaml_variables(vars) + vars.to_h.map do |key, value| + { key: key.to_s, value: value, public: true } + end + end + + def transform_from_yaml_variables(vars) + return vars.stringify_keys if vars.is_a?(Hash) + + vars.to_a.map { |var| [var[:key].to_s, var[:value]] }.to_h + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 86749cda9c7..3459b69bebc 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -123,9 +123,7 @@ module Gitlab end def transform_to_yaml_variables(variables) - variables.to_h.map do |key, value| - { key: key.to_s, value: value, public: true } - end + ::Gitlab::Ci::Variables::Helpers.transform_to_yaml_variables(variables) end end end diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb index 6d18f9070cc..48a1ab23fc2 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb @@ -12,10 +12,9 @@ module Gitlab VALID_NICENESS_LEVELS = %w{none realtime best-effort idle}.freeze attr_accessor :batch, :total_found, :total_cleaned - attr_reader :limit, :dry_run, :niceness, :logger + attr_reader :dry_run, :niceness, :logger - def initialize(limit: nil, dry_run: true, niceness: nil, logger: nil) - @limit = limit + def initialize(dry_run: true, niceness: nil, logger: nil) @dry_run = dry_run @niceness = (niceness || DEFAULT_NICENESS).downcase @logger = logger || Gitlab::AppLogger @@ -31,7 +30,11 @@ module Gitlab batch << artifact_file clean_batch! if batch.full? - break if limit_reached? + + if limit_reached? + log_info("Exiting due to reaching limit of #{limit}.") + break + end end clean_batch! @@ -128,6 +131,10 @@ module Gitlab def log_error(msg, params = {}) logger.error(msg) end + + def limit + ENV['LIMIT']&.to_i + end end end end diff --git a/lib/gitlab/cleanup/orphan_lfs_file_references.rb b/lib/gitlab/cleanup/orphan_lfs_file_references.rb index a6638b2cbc8..99e7550629a 100644 --- a/lib/gitlab/cleanup/orphan_lfs_file_references.rb +++ b/lib/gitlab/cleanup/orphan_lfs_file_references.rb @@ -5,15 +5,14 @@ module Gitlab class OrphanLfsFileReferences include Gitlab::Utils::StrongMemoize - attr_reader :project, :dry_run, :logger, :limit + attr_reader :project, :dry_run, :logger DEFAULT_REMOVAL_LIMIT = 1000 - def initialize(project, dry_run: true, logger: nil, limit: nil) + def initialize(project, dry_run: true, logger: nil) @project = project @dry_run = dry_run @logger = logger || Gitlab::AppLogger - @limit = limit end def run! @@ -67,6 +66,10 @@ module Gitlab def log_info(msg) logger.info("#{'[DRY RUN] ' if dry_run}#{msg}") end + + def limit + ENV['LIMIT']&.to_i + end end end end diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index 4ae75e0db0a..3c71ca9fcf0 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative '../utils' # Gitlab::Utils + module Gitlab module Cluster # @@ -64,6 +66,10 @@ module Gitlab # Blocks will be executed in the order in which they are registered. # class LifecycleEvents + FatalError = Class.new(Exception) # rubocop:disable Lint/InheritException + + USE_FATAL_LIFECYCLE_EVENTS = Gitlab::Utils.to_boolean(ENV.fetch('GITLAB_FATAL_LIFECYCLE_EVENTS', 'true')) + class << self # # Hook registration methods (called from initializers) @@ -111,24 +117,24 @@ module Gitlab # Lifecycle integration methods (called from unicorn.rb, puma.rb, etc.) # def do_worker_start - call(@worker_start_hooks) + call(:worker_start_hooks, @worker_start_hooks) end def do_before_fork - call(@before_fork_hooks) + call(:before_fork_hooks, @before_fork_hooks) end def do_before_graceful_shutdown - call(@master_blackout_period) + call(:master_blackout_period, @master_blackout_period) blackout_seconds = ::Settings.shutdown.blackout_seconds.to_i sleep(blackout_seconds) if blackout_seconds > 0 - call(@master_graceful_shutdown) + call(:master_graceful_shutdown, @master_graceful_shutdown) end def do_before_master_restart - call(@master_restart_hooks) + call(:master_restart_hooks, @master_restart_hooks) end # DEPRECATED @@ -143,8 +149,18 @@ module Gitlab private - def call(hooks) - hooks&.each(&:call) + def call(name, hooks) + return unless hooks + + hooks.each do |hook| + hook.call + rescue => e + Gitlab::ErrorTracking.track_exception(e, type: 'LifecycleEvents', hook: hook) + warn("ERROR: The hook #{name} failed with exception (#{e.class}) \"#{e.message}\".") + + # we consider lifecycle hooks to be fatal errors + raise FatalError, e if USE_FATAL_LIFECYCLE_EVENTS + end end def in_clustered_environment? diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index 822012e0ed6..fd9f58a34f3 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -35,6 +35,10 @@ module Gitlab # regularly rather than rely on OOM behavior for periodic restarting. config.rolling_restart_frequency = 43200 # 12 hours in seconds. + # Spread the rolling restarts out over 1 hour to avoid too many simultaneous + # process startups. + config.rolling_restart_splay_seconds = 0.0..3600.0 # 0 to 1 hour in seconds. + observer = Gitlab::Cluster::PumaWorkerKillerObserver.new config.pre_term = observer.callback end diff --git a/lib/gitlab/composer/cache.rb b/lib/gitlab/composer/cache.rb new file mode 100644 index 00000000000..1f404d63047 --- /dev/null +++ b/lib/gitlab/composer/cache.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'tempfile' + +module Gitlab + module Composer + class Cache + def initialize(project:, name:, last_page_sha: nil) + @project = project + @name = name + @last_page_sha = last_page_sha + end + + def execute + Packages::Composer::Metadatum.transaction do # rubocop: disable CodeReuse/ActiveRecord + # make sure we lock these records at the start + locked_package_metadata + + if locked_package_metadata.any? + mark_pages_for_delete(shas_to_delete) + + create_cache_page! + + # assign the newest page SHA to the packages + locked_package_metadata.update_all(version_cache_sha: version_index.sha) + elsif @last_page_sha + mark_pages_for_delete([@last_page_sha]) + end + end + end + + private + + def mark_pages_for_delete(shas) + Packages::Composer::CacheFile + .with_namespace(@project.namespace) + .with_sha(shas) + .update_all(delete_at: 1.day.from_now) + end + + def create_cache_page! + Packages::Composer::CacheFile + .safe_find_or_create_by!(namespace_id: @project.namespace_id, file_sha256: version_index.sha) do |cache_file| + cache_file.file = CarrierWaveStringFile.new(version_index.to_json) + end + end + + def version_index + @version_index ||= ::Gitlab::Composer::VersionIndex.new(siblings) + end + + def siblings + @siblings ||= locked_package_metadata.map(&:package) + end + + # find all metadata of the package versions and lock it for update + def locked_package_metadata + @locked_package_metadata ||= Packages::Composer::Metadatum + .for_package(@name, @project.id) + .locked_for_update + end + + def shas_to_delete + locked_package_metadata + .map(&:version_cache_sha) + .reject { |sha| sha == version_index.sha } + .compact + end + end + end +end diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb index de9a17a453f..ac0071cdc53 100644 --- a/lib/gitlab/composer/version_index.rb +++ b/lib/gitlab/composer/version_index.rb @@ -20,7 +20,7 @@ module Gitlab private def package_versions_map - @packages.each_with_object({}) do |package, map| + @packages.sort_by(&:version).each_with_object({}) do |package, map| map[package.version] = package_metadata(package) end end diff --git a/lib/gitlab/conan_token.rb b/lib/gitlab/conan_token.rb index 7526c10b608..d03997b4158 100644 --- a/lib/gitlab/conan_token.rb +++ b/lib/gitlab/conan_token.rb @@ -35,7 +35,7 @@ module Gitlab def secret OpenSSL::HMAC.hexdigest( - OpenSSL::Digest::SHA256.new, + OpenSSL::Digest.new('SHA256'), ::Settings.attr_encrypted_db_key_base, HMAC_KEY ) diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 88786ed82ff..8120f2c1243 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -268,17 +268,16 @@ module Gitlab end end - class StringOrNestedArrayOfStringsValidator < NestedArrayOfStringsValidator - def validate_each(record, attribute, value) - unless validate_string_or_nested_array_of_strings(value) - record.errors.add(attribute, 'should be a string or an array containing strings and arrays of strings') - end - end + class StringOrNestedArrayOfStringsValidator < ActiveModel::EachValidator + include LegacyValidationHelpers + include NestedArrayHelpers - private + def validate_each(record, attribute, value) + max_level = options.fetch(:max_level, 1) - def validate_string_or_nested_array_of_strings(values) - validate_string(values) || validate_nested_array_of_strings(values) + unless validate_string(value) || validate_nested_array(value, max_level, &method(:validate_string)) + record.errors.add(attribute, "should be a string or a nested array of strings up to #{max_level} levels deep") + end end end diff --git a/lib/gitlab/config/entry/validators/nested_array_helpers.rb b/lib/gitlab/config/entry/validators/nested_array_helpers.rb new file mode 100644 index 00000000000..9f5d17d74b0 --- /dev/null +++ b/lib/gitlab/config/entry/validators/nested_array_helpers.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Config + module Entry + module Validators + # Include this module to validate deeply nested array of values + # + # class MyNestedValidator < ActiveModel::EachValidator + # include NestedArrayHelpers + # + # def validate_each(record, attribute, value) + # max_depth = options.fetch(:max_depth, 1) + # + # unless validate_nested_array(value, max_depth) { |v| v.is_a?(Integer) } + # record.errors.add(attribute, "is invalid") + # end + # end + # end + # + module NestedArrayHelpers + def validate_nested_array(value, max_depth = 1, &validator_proc) + return false unless value.is_a?(Array) + + validate_nested_array_recursively(value, max_depth, &validator_proc) + end + + private + + # rubocop: disable Performance/RedundantBlockCall + # Disables Rubocop rule for easier readability reasons. + def validate_nested_array_recursively(value, nesting_level, &validator_proc) + return true if validator_proc.call(value) + return false if nesting_level <= 0 + return false unless value.is_a?(Array) + + value.all? do |element| + validate_nested_array_recursively(element, nesting_level - 1, &validator_proc) + end + end + # rubocop: enable Performance/RedundantBlockCall + end + end + end + end +end diff --git a/lib/gitlab/config/loader/yaml.rb b/lib/gitlab/config/loader/yaml.rb index cb3fc49944c..80c9abecd8e 100644 --- a/lib/gitlab/config/loader/yaml.rb +++ b/lib/gitlab/config/loader/yaml.rb @@ -12,8 +12,12 @@ module Gitlab MAX_YAML_SIZE = 1.megabyte MAX_YAML_DEPTH = 100 - def initialize(config) - @config = YAML.safe_load(config, [Symbol], [], true) + def initialize(config, additional_permitted_classes: []) + @config = YAML.safe_load(config, + permitted_classes: [Symbol, *additional_permitted_classes], + permitted_symbols: [], + aliases: true + ) rescue Psych::Exception => e raise Loader::FormatError, e.message end diff --git a/lib/gitlab/crypto_helper.rb b/lib/gitlab/crypto_helper.rb index 87a03d9c58f..4428354642d 100644 --- a/lib/gitlab/crypto_helper.rb +++ b/lib/gitlab/crypto_helper.rb @@ -6,25 +6,44 @@ module Gitlab AES256_GCM_OPTIONS = { algorithm: 'aes-256-gcm', - key: Settings.attr_encrypted_db_key_base_32, - iv: Settings.attr_encrypted_db_key_base_12 + key: Settings.attr_encrypted_db_key_base_32 }.freeze + AES256_GCM_IV_STATIC = Settings.attr_encrypted_db_key_base_12 + def sha256(value) salt = Settings.attr_encrypted_db_key_base_truncated ::Digest::SHA256.base64digest("#{value}#{salt}") end - def aes256_gcm_encrypt(value) - encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value)) - Base64.strict_encode64(encrypted_token) + def aes256_gcm_encrypt(value, nonce: nil) + aes256_gcm_encrypt_using_static_nonce(value) end def aes256_gcm_decrypt(value) return unless value + nonce = Feature.enabled?(:dynamic_nonce_creation) ? dynamic_nonce(value) : AES256_GCM_IV_STATIC encrypted_token = Base64.decode64(value) - Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token)) + decrypted_token = Encryptor.decrypt(AES256_GCM_OPTIONS.merge(value: encrypted_token, iv: nonce)) + decrypted_token + end + + def dynamic_nonce(value) + TokenWithIv.find_nonce_by_hashed_token(value) || AES256_GCM_IV_STATIC + end + + def aes256_gcm_encrypt_using_static_nonce(value) + create_encrypted_token(value, AES256_GCM_IV_STATIC) + end + + def read_only? + Gitlab::Database.read_only? + end + + def create_encrypted_token(value, iv) + encrypted_token = Encryptor.encrypt(AES256_GCM_OPTIONS.merge(value: value, iv: iv)) + Base64.strict_encode64(encrypted_token) end end end diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index d0579a44219..0bf41f9dc0d 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -7,6 +7,10 @@ module Gitlab Gitlab::SafeRequestStore.fetch(:current_application_settings) { ensure_application_settings! } end + def current_application_settings? + Gitlab::SafeRequestStore.exist?(:current_application_settings) || ::ApplicationSetting.current.present? + end + def expire_current_application_settings ::ApplicationSetting.expire Gitlab::SafeRequestStore.delete(:current_application_settings) diff --git a/lib/gitlab/cycle_analytics/summary/deploy.rb b/lib/gitlab/cycle_analytics/summary/deploy.rb index 5125c8e64ee..aaa2554dbfa 100644 --- a/lib/gitlab/cycle_analytics/summary/deploy.rb +++ b/lib/gitlab/cycle_analytics/summary/deploy.rb @@ -15,9 +15,16 @@ module Gitlab private def deployments_count - query = @project.deployments.success.where("created_at >= ?", @from) - query = query.where("created_at <= ?", @to) if @to - query.count + if Feature.enabled?(:query_deploymenys_via_finished_at_in_vsa, default_enabled: :yaml) + DeploymentsFinder + .new(project: @project, finished_after: @from, finished_before: @to, status: :success) + .execute + .count + else + query = @project.deployments.success.where("created_at >= ?", @from) + query = query.where("created_at <= ?", @to) if @to + query.count + end end end end diff --git a/lib/gitlab/danger/base_linter.rb b/lib/gitlab/danger/base_linter.rb deleted file mode 100644 index 898434724bd..00000000000 --- a/lib/gitlab/danger/base_linter.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: true - -require_relative 'title_linting' - -module Gitlab - module Danger - class BaseLinter - MIN_SUBJECT_WORDS_COUNT = 3 - MAX_LINE_LENGTH = 72 - - attr_reader :commit, :problems - - def self.problems_mapping - { - subject_too_short: "The %s must contain at least #{MIN_SUBJECT_WORDS_COUNT} words", - subject_too_long: "The %s may not be longer than #{MAX_LINE_LENGTH} characters", - subject_starts_with_lowercase: "The %s must start with a capital letter", - subject_ends_with_a_period: "The %s must not end with a period" - } - end - - def self.subject_description - 'commit subject' - end - - def initialize(commit) - @commit = commit - @problems = {} - end - - def failed? - problems.any? - end - - def add_problem(problem_key, *args) - @problems[problem_key] = sprintf(self.class.problems_mapping[problem_key], *args) - end - - def lint_subject - if subject_too_short? - add_problem(:subject_too_short, self.class.subject_description) - end - - if subject_too_long? - add_problem(:subject_too_long, self.class.subject_description) - end - - if subject_starts_with_lowercase? - add_problem(:subject_starts_with_lowercase, self.class.subject_description) - end - - if subject_ends_with_a_period? - add_problem(:subject_ends_with_a_period, self.class.subject_description) - end - - self - end - - private - - def subject - TitleLinting.remove_draft_flag(message_parts[0]) - end - - def subject_too_short? - subject.split(' ').length < MIN_SUBJECT_WORDS_COUNT - end - - def subject_too_long? - line_too_long?(subject) - end - - def line_too_long?(line) - line.length > MAX_LINE_LENGTH - end - - def subject_starts_with_lowercase? - return false if ('A'..'Z').cover?(subject[0]) - - first_char = subject.sub(/\A(\[.+\]|\w+:)\s/, '')[0] - first_char_downcased = first_char.downcase - return true unless ('a'..'z').cover?(first_char_downcased) - - first_char.downcase == first_char - end - - def subject_ends_with_a_period? - subject.end_with?('.') - end - - def message_parts - @message_parts ||= commit.message.split("\n", 3) - end - end - end -end diff --git a/lib/gitlab/danger/changelog.rb b/lib/gitlab/danger/changelog.rb deleted file mode 100644 index 4b85775ed98..00000000000 --- a/lib/gitlab/danger/changelog.rb +++ /dev/null @@ -1,92 +0,0 @@ -# frozen_string_literal: true - -require_relative 'title_linting' - -module Gitlab - module Danger - module Changelog - NO_CHANGELOG_LABELS = [ - 'tooling', - 'tooling::pipelines', - 'tooling::workflow', - 'ci-build', - 'meta' - ].freeze - NO_CHANGELOG_CATEGORIES = %i[docs none].freeze - CREATE_CHANGELOG_COMMAND = 'bin/changelog -m %<mr_iid>s "%<mr_title>s"' - CREATE_EE_CHANGELOG_COMMAND = 'bin/changelog --ee -m %<mr_iid>s "%<mr_title>s"' - CHANGELOG_MODIFIED_URL_TEXT = "**CHANGELOG.md was edited.** Please remove the additions and create a CHANGELOG entry.\n\n" - CHANGELOG_MISSING_URL_TEXT = "**[CHANGELOG missing](https://docs.gitlab.com/ee/development/changelog.html)**:\n\n" - - OPTIONAL_CHANGELOG_MESSAGE = <<~MSG - If you want to create a changelog entry for GitLab FOSS, run the following: - - #{CREATE_CHANGELOG_COMMAND} - - If you want to create a changelog entry for GitLab EE, run the following instead: - - #{CREATE_EE_CHANGELOG_COMMAND} - - If this merge request [doesn't need a CHANGELOG entry](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry), feel free to ignore this message. - MSG - - REQUIRED_CHANGELOG_MESSAGE = <<~MSG - To create a changelog entry, run the following: - - #{CREATE_CHANGELOG_COMMAND} - - This merge request requires a changelog entry because it [introduces a database migration](https://docs.gitlab.com/ee/development/changelog.html#what-warrants-a-changelog-entry). - MSG - - def required? - git.added_files.any? { |path| path =~ %r{\Adb/(migrate|post_migrate)/} } - end - alias_method :db_changes?, :required? - - def optional? - categories_need_changelog? && without_no_changelog_label? - end - - def found - @found ||= git.added_files.find { |path| path =~ %r{\A(ee/)?(changelogs/unreleased)(-ee)?/} } - end - - def ee_changelog? - found.start_with?('ee/') - end - - def modified_text - CHANGELOG_MODIFIED_URL_TEXT + - format(OPTIONAL_CHANGELOG_MESSAGE, mr_iid: mr_iid, mr_title: sanitized_mr_title) - end - - def required_text - CHANGELOG_MISSING_URL_TEXT + - format(REQUIRED_CHANGELOG_MESSAGE, mr_iid: mr_iid, mr_title: sanitized_mr_title) - end - - def optional_text - CHANGELOG_MISSING_URL_TEXT + - format(OPTIONAL_CHANGELOG_MESSAGE, mr_iid: mr_iid, mr_title: sanitized_mr_title) - end - - private - - def mr_iid - gitlab.mr_json["iid"] - end - - def sanitized_mr_title - TitleLinting.sanitize_mr_title(gitlab.mr_json["title"]) - end - - def categories_need_changelog? - (helper.changes_by_category.keys - NO_CHANGELOG_CATEGORIES).any? - end - - def without_no_changelog_label? - (gitlab.mr_labels & NO_CHANGELOG_LABELS).empty? - end - end - end -end diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb deleted file mode 100644 index e23f5900433..00000000000 --- a/lib/gitlab/danger/commit_linter.rb +++ /dev/null @@ -1,158 +0,0 @@ -# frozen_string_literal: true - -emoji_checker_path = File.expand_path('emoji_checker', __dir__) -base_linter_path = File.expand_path('base_linter', __dir__) - -if defined?(Rails) - require_dependency(base_linter_path) - require_dependency(emoji_checker_path) -else - require_relative(base_linter_path) - require_relative(emoji_checker_path) -end - -module Gitlab - module Danger - class CommitLinter < BaseLinter - MAX_CHANGED_FILES_IN_COMMIT = 3 - MAX_CHANGED_LINES_IN_COMMIT = 30 - SHORT_REFERENCE_REGEX = %r{([\w\-\/]+)?(?<!`)(#|!|&|%)\d+(?<!`)}.freeze - - def self.problems_mapping - super.merge( - { - separator_missing: "The commit subject and body must be separated by a blank line", - details_too_many_changes: "Commits that change #{MAX_CHANGED_LINES_IN_COMMIT} or more lines across " \ - "at least #{MAX_CHANGED_FILES_IN_COMMIT} files must describe these changes in the commit body", - details_line_too_long: "The commit body should not contain more than #{MAX_LINE_LENGTH} characters per line", - message_contains_text_emoji: "Avoid the use of Markdown Emoji such as `:+1:`. These add limited value " \ - "to the commit message, and are displayed as plain text outside of GitLab", - message_contains_unicode_emoji: "Avoid the use of Unicode Emoji. These add no value to the commit " \ - "message, and may not be displayed properly everywhere", - message_contains_short_reference: "Use full URLs instead of short references (`gitlab-org/gitlab#123` or " \ - "`!123`), as short references are displayed as plain text outside of GitLab" - } - ) - end - - def initialize(commit) - super - - @linted = false - end - - def fixup? - commit.message.start_with?('fixup!', 'squash!') - end - - def suggestion? - commit.message.start_with?('Apply suggestion to') - end - - def merge? - commit.message.start_with?('Merge branch') - end - - def revert? - commit.message.start_with?('Revert "') - end - - def multi_line? - !details.nil? && !details.empty? - end - - def lint - return self if @linted - - @linted = true - lint_subject - lint_separator - lint_details - lint_message - - self - end - - private - - def lint_separator - return self unless separator && !separator.empty? - - add_problem(:separator_missing) - - self - end - - def lint_details - if !multi_line? && many_changes? - add_problem(:details_too_many_changes) - end - - details&.each_line do |line| - line_without_urls = line.strip.gsub(%r{https?://\S+}, '') - - # If the line includes a URL, we'll allow it to exceed MAX_LINE_LENGTH characters, but - # only if the line _without_ the URL does not exceed this limit. - next unless line_too_long?(line_without_urls) - - add_problem(:details_line_too_long) - break - end - - self - end - - def lint_message - if message_contains_text_emoji? - add_problem(:message_contains_text_emoji) - end - - if message_contains_unicode_emoji? - add_problem(:message_contains_unicode_emoji) - end - - if message_contains_short_reference? - add_problem(:message_contains_short_reference) - end - - self - end - - def files_changed - commit.diff_parent.stats[:total][:files] - end - - def lines_changed - commit.diff_parent.stats[:total][:lines] - end - - def many_changes? - files_changed > MAX_CHANGED_FILES_IN_COMMIT && lines_changed > MAX_CHANGED_LINES_IN_COMMIT - end - - def separator - message_parts[1] - end - - def details - message_parts[2]&.gsub(/^Signed-off-by.*$/, '') - end - - def message_contains_text_emoji? - emoji_checker.includes_text_emoji?(commit.message) - end - - def message_contains_unicode_emoji? - emoji_checker.includes_unicode_emoji?(commit.message) - end - - def message_contains_short_reference? - commit.message.match?(SHORT_REFERENCE_REGEX) - end - - def emoji_checker - @emoji_checker ||= Gitlab::Danger::EmojiChecker.new - end - end - end -end diff --git a/lib/gitlab/danger/emoji_checker.rb b/lib/gitlab/danger/emoji_checker.rb deleted file mode 100644 index e31a6ae5011..00000000000 --- a/lib/gitlab/danger/emoji_checker.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -require 'json' - -module Gitlab - module Danger - class EmojiChecker - DIGESTS = File.expand_path('../../../fixtures/emojis/digests.json', __dir__) - ALIASES = File.expand_path('../../../fixtures/emojis/aliases.json', __dir__) - - # A regex that indicates a piece of text _might_ include an Emoji. The regex - # alone is not enough, as we'd match `:foo:bar:baz`. Instead, we use this - # regex to save us from having to check for all possible emoji names when we - # know one definitely is not included. - LIKELY_EMOJI = /:[\+a-z0-9_\-]+:/.freeze - - UNICODE_EMOJI_REGEX = %r{( - [\u{1F300}-\u{1F5FF}] | - [\u{1F1E6}-\u{1F1FF}] | - [\u{2700}-\u{27BF}] | - [\u{1F900}-\u{1F9FF}] | - [\u{1F600}-\u{1F64F}] | - [\u{1F680}-\u{1F6FF}] | - [\u{2600}-\u{26FF}] - )}x.freeze - - def initialize - names = JSON.parse(File.read(DIGESTS)).keys + - JSON.parse(File.read(ALIASES)).keys - - @emoji = names.map { |name| ":#{name}:" } - end - - def includes_text_emoji?(text) - return false unless text.match?(LIKELY_EMOJI) - - @emoji.any? { |emoji| text.include?(emoji) } - end - - def includes_unicode_emoji?(text) - text.match?(UNICODE_EMOJI_REGEX) - end - end - end -end diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb deleted file mode 100644 index 09e013e24b8..00000000000 --- a/lib/gitlab/danger/helper.rb +++ /dev/null @@ -1,273 +0,0 @@ -# frozen_string_literal: true - -require_relative 'teammate' -require_relative 'title_linting' - -module Gitlab - module Danger - module Helper - RELEASE_TOOLS_BOT = 'gitlab-release-tools-bot' - - # Returns a list of all files that have been added, modified or renamed. - # `git.modified_files` might contain paths that already have been renamed, - # so we need to remove them from the list. - # - # Considering these changes: - # - # - A new_file.rb - # - D deleted_file.rb - # - M modified_file.rb - # - R renamed_file_before.rb -> renamed_file_after.rb - # - # it will return - # ``` - # [ 'new_file.rb', 'modified_file.rb', 'renamed_file_after.rb' ] - # ``` - # - # @return [Array<String>] - def all_changed_files - Set.new - .merge(git.added_files.to_a) - .merge(git.modified_files.to_a) - .merge(git.renamed_files.map { |x| x[:after] }) - .subtract(git.renamed_files.map { |x| x[:before] }) - .to_a - .sort - end - - # Returns a string containing changed lines as git diff - # - # Considering changing a line in lib/gitlab/usage_data.rb it will return: - # - # [ "--- a/lib/gitlab/usage_data.rb", - # "+++ b/lib/gitlab/usage_data.rb", - # "+ # Test change", - # "- # Old change" ] - def changed_lines(changed_file) - diff = git.diff_for_file(changed_file) - return [] unless diff - - diff.patch.split("\n").select { |line| %r{^[+-]}.match?(line) } - end - - def all_ee_changes - all_changed_files.grep(%r{\Aee/}) - end - - def ee? - # Support former project name for `dev` and support local Danger run - %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) || Dir.exist?(File.expand_path('../../../ee', __dir__)) - end - - def gitlab_helper - # Unfortunately the following does not work: - # - respond_to?(:gitlab) - # - respond_to?(:gitlab, true) - gitlab - rescue NameError - nil - end - - def release_automation? - gitlab_helper&.mr_author == RELEASE_TOOLS_BOT - end - - def project_name - ee? ? 'gitlab' : 'gitlab-foss' - end - - def markdown_list(items) - list = items.map { |item| "* `#{item}`" }.join("\n") - - if items.size > 10 - "\n<details>\n\n#{list}\n\n</details>\n" - else - list - end - end - - # @return [Hash<String,Array<String>>] - def changes_by_category - all_changed_files.each_with_object(Hash.new { |h, k| h[k] = [] }) do |file, hash| - categories_for_file(file).each { |category| hash[category] << file } - end - end - - # Determines the categories a file is in, e.g., `[:frontend]`, `[:backend]`, or `%i[frontend engineering_productivity]` - # using filename regex and specific change regex if given. - # - # @return Array<Symbol> - def categories_for_file(file) - _, categories = CATEGORIES.find do |key, _| - filename_regex, changes_regex = Array(key) - - found = filename_regex.match?(file) - found &&= changed_lines(file).any? { |changed_line| changes_regex.match?(changed_line) } if changes_regex - - found - end - - Array(categories || :unknown) - end - - # Returns the GFM for a category label, making its best guess if it's not - # a category we know about. - # - # @return[String] - def label_for_category(category) - CATEGORY_LABELS.fetch(category, "~#{category}") - end - - CATEGORY_LABELS = { - docs: "~documentation", # Docs are reviewed along DevOps stages, so don't need roulette for now. - none: "", - qa: "~QA", - test: "~test ~Quality for `spec/features/*`", - engineering_productivity: '~"Engineering Productivity" for CI, Danger', - ci_template: '~"ci::templates"' - }.freeze - # First-match win, so be sure to put more specific regex at the top... - CATEGORIES = { - [%r{usage_data\.rb}, %r{^(\+|-).*\s+(count|distinct_count|estimate_batch_distinct_count)\(.*\)(.*)$}] => [:database, :backend], - - %r{\Adoc/.*(\.(md|png|gif|jpg))\z} => :docs, - %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, - - %r{\A(ee/)?app/(assets|views)/} => :frontend, - %r{\A(ee/)?public/} => :frontend, - %r{\A(ee/)?spec/(javascripts|frontend)/} => :frontend, - %r{\A(ee/)?vendor/assets/} => :frontend, - %r{\A(ee/)?scripts/frontend/} => :frontend, - %r{(\A|/)( - \.babelrc | - \.eslintignore | - \.eslintrc(\.yml)? | - \.nvmrc | - \.prettierignore | - \.prettierrc | - \.scss-lint.yml | - \.stylelintrc | - \.haml-lint.yml | - \.haml-lint_todo.yml | - babel\.config\.js | - jest\.config\.js | - package\.json | - yarn\.lock | - config/.+\.js - )\z}x => :frontend, - - %r{(\A|/)( - \.gitlab/ci/frontend\.gitlab-ci\.yml - )\z}x => %i[frontend engineering_productivity], - - %r{\A(ee/)?db/(?!fixtures)[^/]+} => :database, - %r{\A(ee/)?lib/gitlab/(database|background_migration|sql|github_import)(/|\.rb)} => :database, - %r{\A(app/models/project_authorization|app/services/users/refresh_authorized_projects_service)(/|\.rb)} => :database, - %r{\A(ee/)?app/finders/} => :database, - %r{\Arubocop/cop/migration(/|\.rb)} => :database, - - %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :engineering_productivity, - %r{\A\.codeclimate\.yml\z} => :engineering_productivity, - %r{\Alefthook.yml\z} => :engineering_productivity, - %r{\A\.editorconfig\z} => :engineering_productivity, - %r{Dangerfile\z} => :engineering_productivity, - %r{\A(ee/)?(danger/|lib/gitlab/danger/)} => :engineering_productivity, - %r{\A(ee/)?scripts/} => :engineering_productivity, - %r{\Atooling/} => :engineering_productivity, - %r{(CODEOWNERS)} => :engineering_productivity, - %r{(tests.yml)} => :engineering_productivity, - - %r{\Alib/gitlab/ci/templates} => :ci_template, - - %r{\A(ee/)?spec/features/} => :test, - %r{\A(ee/)?spec/support/shared_examples/features/} => :test, - %r{\A(ee/)?spec/support/shared_contexts/features/} => :test, - %r{\A(ee/)?spec/support/helpers/features/} => :test, - - %r{\A(ee/)?app/(?!assets|views)[^/]+} => :backend, - %r{\A(ee/)?(bin|config|generator_templates|lib|rubocop)/} => :backend, - %r{\A(ee/)?spec/} => :backend, - %r{\A(ee/)?vendor/} => :backend, - %r{\A(Gemfile|Gemfile.lock|Rakefile)\z} => :backend, - %r{\A[A-Z_]+_VERSION\z} => :backend, - %r{\A\.rubocop((_manual)?_todo)?\.yml\z} => :backend, - %r{\Afile_hooks/} => :backend, - - %r{\A(ee/)?qa/} => :qa, - - # Files that don't fit into any category are marked with :none - %r{\A(ee/)?changelogs/} => :none, - %r{\Alocale/gitlab\.pot\z} => :none, - %r{\Adata/whats_new/} => :none, - - # GraphQL auto generated doc files and schema - %r{\Adoc/api/graphql/reference/} => :backend, - - # Fallbacks in case the above patterns miss anything - %r{\.rb\z} => :backend, - %r{( - \.(md|txt)\z | - \.markdownlint\.json - )}x => :none, # To reinstate roulette for documentation, set to `:docs`. - %r{\.js\z} => :frontend - }.freeze - - def new_teammates(usernames) - usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) } - end - - def draft_mr? - return false unless gitlab_helper - - TitleLinting.has_draft_flag?(gitlab_helper.mr_json['title']) - end - - def security_mr? - return false unless gitlab_helper - - gitlab_helper.mr_json['web_url'].include?('/gitlab-org/security/') - end - - def cherry_pick_mr? - return false unless gitlab_helper - - /cherry[\s-]*pick/i.match?(gitlab_helper.mr_json['title']) - end - - def stable_branch? - return false unless gitlab_helper - - /\A\d+-\d+-stable-ee/i.match?(gitlab_helper.mr_json['target_branch']) - end - - def mr_has_labels?(*labels) - return false unless gitlab_helper - - labels = labels.flatten.uniq - (labels & gitlab_helper.mr_labels) == labels - end - - def labels_list(labels, sep: ', ') - labels.map { |label| %Q{~"#{label}"} }.join(sep) - end - - def prepare_labels_for_mr(labels) - return '' unless labels.any? - - "/label #{labels_list(labels, sep: ' ')}" - end - - def changed_files(regex) - all_changed_files.grep(regex) - end - - def has_database_scoped_labels?(current_mr_labels) - current_mr_labels.any? { |label| label.start_with?('database::') } - end - - def has_ci_changes? - changed_files(%r{\A(\.gitlab-ci\.yml|\.gitlab/ci/)}).any? - end - end - end -end diff --git a/lib/gitlab/danger/merge_request_linter.rb b/lib/gitlab/danger/merge_request_linter.rb deleted file mode 100644 index ed354bfc68d..00000000000 --- a/lib/gitlab/danger/merge_request_linter.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -base_linter_path = File.expand_path('base_linter', __dir__) - -if defined?(Rails) - require_dependency(base_linter_path) -else - require_relative(base_linter_path) -end - -module Gitlab - module Danger - class MergeRequestLinter < BaseLinter - alias_method :lint, :lint_subject - - def self.subject_description - 'merge request title' - end - - def self.mr_run_options_regex - [ - 'RUN AS-IF-FOSS', - 'UPDATE CACHE', - 'RUN ALL RSPEC', - 'SKIP RSPEC FAIL-FAST' - ].join('|') - end - - private - - def subject - super.gsub(/\[?(#{self.class.mr_run_options_regex})\]?/, '').strip - end - end - end -end diff --git a/lib/gitlab/danger/request_helper.rb b/lib/gitlab/danger/request_helper.rb deleted file mode 100644 index 06da4ed9ad3..00000000000 --- a/lib/gitlab/danger/request_helper.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require 'net/http' -require 'json' - -module Gitlab - module Danger - module RequestHelper - HTTPError = Class.new(RuntimeError) - - # @param [String] url - def self.http_get_json(url) - rsp = Net::HTTP.get_response(URI.parse(url)) - - unless rsp.is_a?(Net::HTTPOK) - raise HTTPError, "Failed to read #{url}: #{rsp.code} #{rsp.message}" - end - - JSON.parse(rsp.body) - end - end - end -end diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb deleted file mode 100644 index 21feda2cf20..00000000000 --- a/lib/gitlab/danger/roulette.rb +++ /dev/null @@ -1,169 +0,0 @@ -# frozen_string_literal: true - -require_relative 'teammate' -require_relative 'request_helper' unless defined?(Gitlab::Danger::RequestHelper) -require_relative 'weightage/reviewers' -require_relative 'weightage/maintainers' - -module Gitlab - module Danger - module Roulette - ROULETTE_DATA_URL = 'https://gitlab-org.gitlab.io/gitlab-roulette/roulette.json' - HOURS_WHEN_PERSON_CAN_BE_PICKED = (6..14).freeze - - INCLUDE_TIMEZONE_FOR_CATEGORY = { - database: false - }.freeze - - Spin = Struct.new(:category, :reviewer, :maintainer, :optional_role, :timezone_experiment) - - def team_mr_author - team.find { |person| person.username == mr_author_username } - end - - # Assigns GitLab team members to be reviewer and maintainer - # for each change category that a Merge Request contains. - # - # @return [Array<Spin>] - def spin(project, categories, timezone_experiment: false) - spins = categories.sort.map do |category| - including_timezone = INCLUDE_TIMEZONE_FOR_CATEGORY.fetch(category, timezone_experiment) - - spin_for_category(project, category, timezone_experiment: including_timezone) - end - - backend_spin = spins.find { |spin| spin.category == :backend } - - spins.each do |spin| - including_timezone = INCLUDE_TIMEZONE_FOR_CATEGORY.fetch(spin.category, timezone_experiment) - case spin.category - when :qa - # MR includes QA changes, but also other changes, and author isn't an SET - if categories.size > 1 && !team_mr_author&.reviewer?(project, spin.category, []) - spin.optional_role = :maintainer - end - when :test - spin.optional_role = :maintainer - - if spin.reviewer.nil? - # Fetch an already picked backend reviewer, or pick one otherwise - spin.reviewer = backend_spin&.reviewer || spin_for_category(project, :backend, timezone_experiment: including_timezone).reviewer - end - when :engineering_productivity - if spin.maintainer.nil? - # Fetch an already picked backend maintainer, or pick one otherwise - spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer - end - when :ci_template - if spin.maintainer.nil? - # Fetch an already picked backend maintainer, or pick one otherwise - spin.maintainer = backend_spin&.maintainer || spin_for_category(project, :backend, timezone_experiment: including_timezone).maintainer - end - end - end - - spins - end - - # Looks up the current list of GitLab team members and parses it into a - # useful form - # - # @return [Array<Teammate>] - def team - @team ||= - begin - data = Gitlab::Danger::RequestHelper.http_get_json(ROULETTE_DATA_URL) - data.map { |hash| ::Gitlab::Danger::Teammate.new(hash) } - rescue JSON::ParserError - raise "Failed to parse JSON response from #{ROULETTE_DATA_URL}" - end - end - - # Like +team+, but only returns teammates in the current project, based on - # project_name. - # - # @return [Array<Teammate>] - def project_team(project_name) - team.select { |member| member.in_project?(project_name) } - rescue => err - warn("Reviewer roulette failed to load team data: #{err.message}") - [] - end - - # Known issue: If someone is rejected due to OOO, and then becomes not OOO, the - # selection will change on next spin - # @param [Array<Teammate>] people - def spin_for_person(people, random:, timezone_experiment: false) - shuffled_people = people.shuffle(random: random) - - if timezone_experiment - shuffled_people.find(&method(:valid_person_with_timezone?)) - else - shuffled_people.find(&method(:valid_person?)) - end - end - - private - - # @param [Teammate] person - # @return [Boolean] - def valid_person?(person) - !mr_author?(person) && person.available - end - - # @param [Teammate] person - # @return [Boolean] - def valid_person_with_timezone?(person) - valid_person?(person) && HOURS_WHEN_PERSON_CAN_BE_PICKED.cover?(person.local_hour) - end - - # @param [Teammate] person - # @return [Boolean] - def mr_author?(person) - person.username == mr_author_username - end - - def mr_author_username - helper.gitlab_helper&.mr_author || `whoami` - end - - def mr_source_branch - return `git rev-parse --abbrev-ref HEAD` unless helper.gitlab_helper&.mr_json - - helper.gitlab_helper.mr_json['source_branch'] - end - - def mr_labels - helper.gitlab_helper&.mr_labels || [] - end - - def new_random(seed) - Random.new(Digest::MD5.hexdigest(seed).to_i(16)) - end - - def spin_role_for_category(team, role, project, category) - team.select do |member| - member.public_send("#{role}?", project, category, mr_labels) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def spin_for_category(project, category, timezone_experiment: false) - team = project_team(project) - reviewers, traintainers, maintainers = - %i[reviewer traintainer maintainer].map do |role| - spin_role_for_category(team, role, project, category) - end - - random = new_random(mr_source_branch) - - weighted_reviewers = Weightage::Reviewers.new(reviewers, traintainers).execute - weighted_maintainers = Weightage::Maintainers.new(maintainers).execute - - reviewer = spin_for_person(weighted_reviewers, random: random, timezone_experiment: timezone_experiment) - maintainer = spin_for_person(weighted_maintainers, random: random, timezone_experiment: timezone_experiment) - - Spin.new(category, reviewer, maintainer, false, timezone_experiment) - end - end - end -end diff --git a/lib/gitlab/danger/sidekiq_queues.rb b/lib/gitlab/danger/sidekiq_queues.rb deleted file mode 100644 index 726b6134abf..00000000000 --- a/lib/gitlab/danger/sidekiq_queues.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Danger - module SidekiqQueues - def changed_queue_files - @changed_queue_files ||= git.modified_files.grep(%r{\A(ee/)?app/workers/all_queues\.yml}) - end - - def added_queue_names - @added_queue_names ||= new_queues.keys - old_queues.keys - end - - def changed_queue_names - @changed_queue_names ||= - (new_queues.values_at(*old_queues.keys) - old_queues.values) - .compact.map { |queue| queue[:name] } - end - - private - - def old_queues - @old_queues ||= queues_for(gitlab.base_commit) - end - - def new_queues - @new_queues ||= queues_for(gitlab.head_commit) - end - - def queues_for(branch) - changed_queue_files - .flat_map { |file| YAML.safe_load(`git show #{branch}:#{file}`, permitted_classes: [Symbol]) } - .to_h { |queue| [queue[:name], queue] } - end - end - end -end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb deleted file mode 100644 index 911b84d93ec..00000000000 --- a/lib/gitlab/danger/teammate.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Danger - class Teammate - attr_reader :options, :username, :name, :role, :projects, :available, :hungry, :reduced_capacity, :tz_offset_hours - - # The options data are produced by https://gitlab.com/gitlab-org/gitlab-roulette/-/blob/master/lib/team_member.rb - def initialize(options = {}) - @options = options - @username = options['username'] - @name = options['name'] - @markdown_name = options['markdown_name'] - @role = options['role'] - @projects = options['projects'] - @available = options['available'] - @hungry = options['hungry'] - @reduced_capacity = options['reduced_capacity'] - @tz_offset_hours = options['tz_offset_hours'] - end - - def to_h - options - end - - def ==(other) - return false unless other.respond_to?(:username) - - other.username == username - end - - def in_project?(name) - projects&.has_key?(name) - end - - def reviewer?(project, category, labels) - has_capability?(project, category, :reviewer, labels) - end - - def traintainer?(project, category, labels) - has_capability?(project, category, :trainee_maintainer, labels) - end - - def maintainer?(project, category, labels) - has_capability?(project, category, :maintainer, labels) - end - - def markdown_name(author: nil) - "#{@markdown_name} (#{utc_offset_text(author)})" - end - - def local_hour - (Time.now.utc + tz_offset_hours * 3600).hour - end - - protected - - def floored_offset_hours - floored_offset = tz_offset_hours.floor(0) - - floored_offset == tz_offset_hours ? floored_offset : tz_offset_hours - end - - private - - def utc_offset_text(author = nil) - offset_text = - if floored_offset_hours >= 0 - "UTC+#{floored_offset_hours}" - else - "UTC#{floored_offset_hours}" - end - - return offset_text unless author - - "#{offset_text}, #{offset_diff_compared_to_author(author)}" - end - - def offset_diff_compared_to_author(author) - diff = floored_offset_hours - author.floored_offset_hours - return "same timezone as `@#{author.username}`" if diff == 0 - - ahead_or_behind = diff < 0 ? 'behind' : 'ahead of' - pluralized_hours = pluralize(diff.abs, 'hour', 'hours') - - "#{pluralized_hours} #{ahead_or_behind} `@#{author.username}`" - end - - def has_capability?(project, category, kind, labels) - case category - when :test - area = role[/Software Engineer in Test(?:.*?, (\w+))/, 1] - - area && labels.any?("devops::#{area.downcase}") if kind == :reviewer - when :engineering_productivity - return false unless role[/Engineering Productivity/] - return true if kind == :reviewer - return true if capabilities(project).include?("#{kind} engineering_productivity") - - capabilities(project).include?("#{kind} backend") - else - capabilities(project).include?("#{kind} #{category}") - end - end - - def capabilities(project) - Array(projects.fetch(project, [])) - end - - def pluralize(count, singular, plural) - word = count == 1 || count.to_s =~ /^1(\.0+)?$/ ? singular : plural - - "#{count || 0} #{word}" - end - end - end -end diff --git a/lib/gitlab/danger/title_linting.rb b/lib/gitlab/danger/title_linting.rb deleted file mode 100644 index db1ccaaf9a9..00000000000 --- a/lib/gitlab/danger/title_linting.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Danger - module TitleLinting - DRAFT_REGEX = /\A*#{Regexp.union(/(?i)(\[WIP\]\s*|WIP:\s*|WIP$)/, /(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft$)/)}+\s*/i.freeze - - module_function - - def sanitize_mr_title(title) - remove_draft_flag(title).gsub(/`/, '\\\`') - end - - def remove_draft_flag(title) - title.gsub(DRAFT_REGEX, '') - end - - def has_draft_flag?(title) - DRAFT_REGEX.match?(title) - end - end - end -end diff --git a/lib/gitlab/danger/weightage.rb b/lib/gitlab/danger/weightage.rb deleted file mode 100644 index 67fade27573..00000000000 --- a/lib/gitlab/danger/weightage.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Danger - module Weightage - CAPACITY_MULTIPLIER = 2 # change this number to change what it means to be a reduced capacity reviewer 1/this number - BASE_REVIEWER_WEIGHT = 1 - end - end -end diff --git a/lib/gitlab/danger/weightage/maintainers.rb b/lib/gitlab/danger/weightage/maintainers.rb deleted file mode 100644 index cc0eb370e7a..00000000000 --- a/lib/gitlab/danger/weightage/maintainers.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -require_relative '../weightage' - -module Gitlab - module Danger - module Weightage - class Maintainers - def initialize(maintainers) - @maintainers = maintainers - end - - def execute - maintainers.each_with_object([]) do |maintainer, weighted_maintainers| - add_weighted_reviewer(weighted_maintainers, maintainer, BASE_REVIEWER_WEIGHT) - end - end - - private - - attr_reader :maintainers - - def add_weighted_reviewer(reviewers, reviewer, weight) - if reviewer.reduced_capacity - reviewers.fill(reviewer, reviewers.size, weight) - else - reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER) - end - end - end - end - end -end diff --git a/lib/gitlab/danger/weightage/reviewers.rb b/lib/gitlab/danger/weightage/reviewers.rb deleted file mode 100644 index c8019be716e..00000000000 --- a/lib/gitlab/danger/weightage/reviewers.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -require_relative '../weightage' - -module Gitlab - module Danger - module Weightage - # Weights after (current multiplier of 2) - # - # +------------------------------+--------------------------------+ - # | reviewer type | weight(times in reviewer pool) | - # +------------------------------+--------------------------------+ - # | reduced capacity reviewer | 1 | - # | reviewer | 2 | - # | hungry reviewer | 4 | - # | reduced capacity traintainer | 3 | - # | traintainer | 6 | - # | hungry traintainer | 8 | - # +------------------------------+--------------------------------+ - # - class Reviewers - DEFAULT_REVIEWER_WEIGHT = CAPACITY_MULTIPLIER * BASE_REVIEWER_WEIGHT - TRAINTAINER_WEIGHT = 3 - - def initialize(reviewers, traintainers) - @reviewers = reviewers - @traintainers = traintainers - end - - def execute - # TODO: take CODEOWNERS into account? - # https://gitlab.com/gitlab-org/gitlab/issues/26723 - - weighted_reviewers + weighted_traintainers - end - - private - - attr_reader :reviewers, :traintainers - - def weighted_reviewers - reviewers.each_with_object([]) do |reviewer, total_reviewers| - add_weighted_reviewer(total_reviewers, reviewer, BASE_REVIEWER_WEIGHT) - end - end - - def weighted_traintainers - traintainers.each_with_object([]) do |reviewer, total_traintainers| - add_weighted_reviewer(total_traintainers, reviewer, TRAINTAINER_WEIGHT) - end - end - - def add_weighted_reviewer(reviewers, reviewer, weight) - if reviewer.reduced_capacity - reviewers.fill(reviewer, reviewers.size, weight) - elsif reviewer.hungry - reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER + DEFAULT_REVIEWER_WEIGHT) - else - reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER) - end - end - end - end - end -end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index e6702c5a38b..e17bd25e57e 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -82,7 +82,8 @@ module Gitlab id: runner.id, description: runner.description, active: runner.active?, - is_shared: runner.instance_type? + is_shared: runner.instance_type?, + tags: runner.tags&.map(&:name) } end end diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 14facd6b1d4..3036bc57ca5 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -13,7 +13,7 @@ module Gitlab user: pipeline.user.try(:hook_attrs), project: pipeline.project.hook_attrs(backward: false), commit: pipeline.commit.try(:hook_attrs), - builds: pipeline.builds.map(&method(:build_hook_attrs)) + builds: pipeline.builds.latest.map(&method(:build_hook_attrs)) } end @@ -76,7 +76,8 @@ module Gitlab id: runner.id, description: runner.description, active: runner.active?, - is_shared: runner.instance_type? + is_shared: runner.instance_type?, + tags: runner.tags&.map(&:name) } end end diff --git a/lib/gitlab/database/consistency.rb b/lib/gitlab/database/consistency.rb new file mode 100644 index 00000000000..b7d06a26ddb --- /dev/null +++ b/lib/gitlab/database/consistency.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Database + ## + # This class is used to make it possible to ensure read consistency in + # GitLab EE without the need of overriding a lot of methods / classes / + # classs. + # + # This is a CE class that does nothing in CE, because database load + # balancing is EE-only feature, but you can still use it in CE. It will + # start ensuring read consistency once it is overridden in EE. + # + # Using this class in CE helps to avoid creeping discrepancy between CE / + # EE only to force usage of the primary database in EE. + # + class Consistency + ## + # In CE there is no database load balancing, so all reads are expected to + # be consistent by the ACID guarantees of a single PostgreSQL instance. + # + # This method is overridden in EE. + # + def self.with_read_consistency(&block) + yield + end + end + end +end + +::Gitlab::Database::Consistency.singleton_class.prepend_if_ee('EE::Gitlab::Database::Consistency') diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb new file mode 100644 index 00000000000..f20a9b30fa7 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/v2.rb @@ -0,0 +1,219 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module V2 + include Gitlab::Database::MigrationHelpers + + # Renames a column without requiring downtime. + # + # Concurrent renames work by using database triggers to ensure both the + # old and new column are in sync. However, this method will _not_ remove + # the triggers or the old column automatically; this needs to be done + # manually in a post-deployment migration. This can be done using the + # method `cleanup_concurrent_column_rename`. + # + # table - The name of the database table containing the column. + # old_column - The old column name. + # new_column - The new column name. + # type - The type of the new column. If no type is given the old column's + # type is used. + # batch_column_name - option is for tables without primary key, in this + # case another unique integer column can be used. Example: :user_id + def rename_column_concurrently(table, old_column, new_column, type: nil, batch_column_name: :id) + setup_renamed_column(__callee__, table, old_column, new_column, type, batch_column_name) + + with_lock_retries do + install_bidirectional_triggers(table, old_column, new_column) + end + end + + # Reverses operations performed by rename_column_concurrently. + # + # This method takes care of removing previously installed triggers as well + # as removing the new column. + # + # table - The name of the database table. + # old_column - The name of the old column. + # new_column - The name of the new column. + def undo_rename_column_concurrently(table, old_column, new_column) + teardown_rename_mechanism(table, old_column, new_column, column_to_remove: new_column) + end + + # Cleans up a concurrent column name. + # + # This method takes care of removing previously installed triggers as well + # as removing the old column. + # + # table - The name of the database table. + # old_column - The name of the old column. + # new_column - The name of the new column. + def cleanup_concurrent_column_rename(table, old_column, new_column) + teardown_rename_mechanism(table, old_column, new_column, column_to_remove: old_column) + end + + # Reverses the operations performed by cleanup_concurrent_column_rename. + # + # This method adds back the old_column removed + # by cleanup_concurrent_column_rename. + # It also adds back the triggers that are removed + # by cleanup_concurrent_column_rename. + # + # table - The name of the database table containing the column. + # old_column - The old column name. + # new_column - The new column name. + # type - The type of the old column. If no type is given the new column's + # type is used. + # batch_column_name - option is for tables without primary key, in this + # case another unique integer column can be used. Example: :user_id + # + def undo_cleanup_concurrent_column_rename(table, old_column, new_column, type: nil, batch_column_name: :id) + setup_renamed_column(__callee__, table, new_column, old_column, type, batch_column_name) + + with_lock_retries do + install_bidirectional_triggers(table, old_column, new_column) + end + end + + private + + def setup_renamed_column(calling_operation, table, old_column, new_column, type, batch_column_name) + if transaction_open? + raise "#{calling_operation} can not be run inside a transaction" + end + + column = columns(table).find { |column| column.name == old_column.to_s } + + unless column + raise "Column #{old_column} does not exist on #{table}" + end + + if column.default + raise "#{calling_operation} does not currently support columns with default values" + end + + unless column_exists?(table, batch_column_name) + raise "Column #{batch_column_name} does not exist on #{table}" + end + + check_trigger_permissions!(table) + + unless column_exists?(table, new_column) + create_column_from(table, old_column, new_column, type: type, batch_column_name: batch_column_name) + end + end + + def teardown_rename_mechanism(table, old_column, new_column, column_to_remove:) + return unless column_exists?(table, column_to_remove) + + with_lock_retries do + check_trigger_permissions!(table) + + remove_bidirectional_triggers(table, old_column, new_column) + + remove_column(table, column_to_remove) + end + end + + def install_bidirectional_triggers(table, old_column, new_column) + insert_trigger_name, update_old_trigger_name, update_new_trigger_name = + bidirectional_trigger_names(table, old_column, new_column) + + quoted_table = quote_table_name(table) + quoted_old = quote_column_name(old_column) + quoted_new = quote_column_name(new_column) + + create_insert_trigger(insert_trigger_name, quoted_table, quoted_old, quoted_new) + create_update_trigger(update_old_trigger_name, quoted_table, quoted_new, quoted_old) + create_update_trigger(update_new_trigger_name, quoted_table, quoted_old, quoted_new) + end + + def remove_bidirectional_triggers(table, old_column, new_column) + insert_trigger_name, update_old_trigger_name, update_new_trigger_name = + bidirectional_trigger_names(table, old_column, new_column) + + quoted_table = quote_table_name(table) + + drop_trigger(insert_trigger_name, quoted_table) + drop_trigger(update_old_trigger_name, quoted_table) + drop_trigger(update_new_trigger_name, quoted_table) + end + + def bidirectional_trigger_names(table, old_column, new_column) + %w[insert update_old update_new].map do |operation| + 'trigger_' + Digest::SHA256.hexdigest("#{table}_#{old_column}_#{new_column}_#{operation}").first(12) + end + end + + def function_name_for_trigger(trigger_name) + "function_for_#{trigger_name}" + end + + def create_insert_trigger(trigger_name, quoted_table, quoted_old_column, quoted_new_column) + function_name = function_name_for_trigger(trigger_name) + + execute(<<~SQL) + CREATE OR REPLACE FUNCTION #{function_name}() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + IF NEW.#{quoted_old_column} IS NULL AND NEW.#{quoted_new_column} IS NOT NULL THEN + NEW.#{quoted_old_column} = NEW.#{quoted_new_column}; + END IF; + + IF NEW.#{quoted_new_column} IS NULL AND NEW.#{quoted_old_column} IS NOT NULL THEN + NEW.#{quoted_new_column} = NEW.#{quoted_old_column}; + END IF; + + RETURN NEW; + END + $$; + + DROP TRIGGER IF EXISTS #{trigger_name} + ON #{quoted_table}; + + CREATE TRIGGER #{trigger_name} + BEFORE INSERT ON #{quoted_table} + FOR EACH ROW EXECUTE FUNCTION #{function_name}(); + SQL + end + + def create_update_trigger(trigger_name, quoted_table, quoted_source_column, quoted_target_column) + function_name = function_name_for_trigger(trigger_name) + + execute(<<~SQL) + CREATE OR REPLACE FUNCTION #{function_name}() + RETURNS trigger + LANGUAGE plpgsql + AS $$ + BEGIN + NEW.#{quoted_target_column} := NEW.#{quoted_source_column}; + RETURN NEW; + END + $$; + + DROP TRIGGER IF EXISTS #{trigger_name} + ON #{quoted_table}; + + CREATE TRIGGER #{trigger_name} + BEFORE UPDATE OF #{quoted_source_column} ON #{quoted_table} + FOR EACH ROW EXECUTE FUNCTION #{function_name}(); + SQL + end + + def drop_trigger(trigger_name, quoted_table) + function_name = function_name_for_trigger(trigger_name) + + execute(<<~SQL) + DROP TRIGGER IF EXISTS #{trigger_name} + ON #{quoted_table}; + + DROP FUNCTION IF EXISTS #{function_name}; + SQL + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb new file mode 100644 index 00000000000..959028ce00b --- /dev/null +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class Instrumentation + attr_reader :observations + + def initialize(observers = ::Gitlab::Database::Migrations::Observers.all_observers) + @observers = observers + @observations = [] + end + + def observe(migration, &block) + observation = Observation.new(migration) + observation.success = true + + exception = nil + + on_each_observer { |observer| observer.before } + + observation.walltime = Benchmark.realtime do + yield + rescue => e + exception = e + observation.success = false + end + + on_each_observer { |observer| observer.after } + on_each_observer { |observer| observer.record(observation) } + + record_observation(observation) + + raise exception if exception + + observation + end + + private + + attr_reader :observers + + def record_observation(observation) + @observations << observation + end + + def on_each_observer(&block) + observers.each do |observer| + yield observer + rescue => e + Gitlab::AppLogger.error("Migration observer #{observer.class} failed with: #{e}") + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb new file mode 100644 index 00000000000..518c2c560d2 --- /dev/null +++ b/lib/gitlab/database/migrations/observation.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + Observation = Struct.new( + :migration, + :walltime, + :success, + :total_database_size_change + ) + end + end +end diff --git a/lib/gitlab/database/migrations/observers.rb b/lib/gitlab/database/migrations/observers.rb new file mode 100644 index 00000000000..4b931d3c19c --- /dev/null +++ b/lib/gitlab/database/migrations/observers.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + def self.all_observers + [ + TotalDatabaseSizeChange.new + ] + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/observers/migration_observer.rb b/lib/gitlab/database/migrations/observers/migration_observer.rb new file mode 100644 index 00000000000..9bfbf35887d --- /dev/null +++ b/lib/gitlab/database/migrations/observers/migration_observer.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + class MigrationObserver + attr_reader :connection + + def initialize + @connection = ActiveRecord::Base.connection + end + + def before + # implement in subclass + end + + def after + # implement in subclass + end + + def record(observation) + raise NotImplementedError, 'implement in subclass' + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/observers/total_database_size_change.rb b/lib/gitlab/database/migrations/observers/total_database_size_change.rb new file mode 100644 index 00000000000..0b76b0bef5e --- /dev/null +++ b/lib/gitlab/database/migrations/observers/total_database_size_change.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module Observers + class TotalDatabaseSizeChange < MigrationObserver + def before + @size_before = get_total_database_size + end + + def after + @size_after = get_total_database_size + end + + def record(observation) + return unless @size_after && @size_before + + observation.total_database_size_change = @size_after - @size_before + end + + private + + def get_total_database_size + connection.execute("select pg_database_size(current_database())").first['pg_database_size'] + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 686dda80207..f4cf576dda7 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -164,8 +164,8 @@ module Gitlab "this could indicate the previous partitioning migration has been rolled back." end - Gitlab::BackgroundMigration.steal(MIGRATION_CLASS_NAME) do |raw_arguments| - JobArguments.from_array(raw_arguments).source_table_name == table_name.to_s + Gitlab::BackgroundMigration.steal(MIGRATION_CLASS_NAME) do |background_job| + JobArguments.from_array(background_job.args.second).source_table_name == table_name.to_s end primary_key = connection.primary_key(table_name) diff --git a/lib/gitlab/diff/char_diff.rb b/lib/gitlab/diff/char_diff.rb new file mode 100644 index 00000000000..c8bb39e9f5d --- /dev/null +++ b/lib/gitlab/diff/char_diff.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module Diff + class CharDiff + include Gitlab::Utils::StrongMemoize + + def initialize(old_string, new_string) + @old_string = old_string.to_s + @new_string = new_string.to_s + @changes = [] + end + + def generate_diff + @changes = diff_match_patch.diff_main(@old_string, @new_string) + diff_match_patch.diff_cleanupSemantic(@changes) + + @changes + end + + def changed_ranges(offset: 0) + old_diffs = [] + new_diffs = [] + new_pointer = old_pointer = offset + + generate_diff.each do |(action, content)| + content_size = content.size + + if action == :equal + new_pointer += content_size + old_pointer += content_size + end + + if action == :delete + old_diffs << (old_pointer..(old_pointer + content_size - 1)) + old_pointer += content_size + end + + if action == :insert + new_diffs << (new_pointer..(new_pointer + content_size - 1)) + new_pointer += content_size + end + end + + [old_diffs, new_diffs] + end + + def to_html + @changes.map do |op, text| + %{<span class="#{html_class_names(op)}">#{ERB::Util.html_escape(text)}</span>} + end.join.html_safe + end + + private + + def diff_match_patch + strong_memoize(:diff_match_patch) { DiffMatchPatch.new } + end + + def html_class_names(operation) + class_names = ['idiff'] + + case operation + when :insert + class_names << 'addition' + when :delete + class_names << 'deletion' + end + + class_names.join(' ') + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 8f4f8febec0..627abfbfe7e 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -117,7 +117,7 @@ module Gitlab end def sort_diffs(diffs) - return diffs unless Feature.enabled?(:sort_diffs, project, default_enabled: false) + return diffs unless Feature.enabled?(:sort_diffs, project, default_enabled: :yaml) Gitlab::Diff::FileCollectionSorter.new(diffs).sort end diff --git a/lib/gitlab/diff/file_collection_sorter.rb b/lib/gitlab/diff/file_collection_sorter.rb index 94626875580..7b099543c83 100644 --- a/lib/gitlab/diff/file_collection_sorter.rb +++ b/lib/gitlab/diff/file_collection_sorter.rb @@ -3,6 +3,10 @@ module Gitlab module Diff class FileCollectionSorter + B_FOLLOWS_A = 1 + A_FOLLOWS_B = -1 + EQUIVALENT = 0 + attr_reader :diffs def initialize(diffs) @@ -29,14 +33,16 @@ module Gitlab a_part = a_parts.shift b_part = b_parts.shift - return 1 if a_parts.size < b_parts.size && a_parts.empty? - return -1 if a_parts.size > b_parts.size && b_parts.empty? + return B_FOLLOWS_A if a_parts.size < b_parts.size && a_parts.empty? + return A_FOLLOWS_B if a_parts.size > b_parts.size && b_parts.empty? comparison = a_part <=> b_part - return comparison unless comparison == 0 + return comparison unless comparison == EQUIVALENT + return compare_path_parts(a_parts, b_parts) if a_parts.any? && b_parts.any? - compare_path_parts(a_parts, b_parts) + # If A and B have the same name (e.g. symlink change), they are identical so return 0 + EQUIVALENT end end end diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index a5259079345..035084d4861 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -3,12 +3,13 @@ module Gitlab module Diff class Highlight - attr_reader :diff_file, :diff_lines, :raw_lines, :repository + attr_reader :diff_file, :diff_lines, :raw_lines, :repository, :project delegate :old_path, :new_path, :old_sha, :new_sha, to: :diff_file, prefix: :diff def initialize(diff_lines, repository: nil) @repository = repository + @project = repository&.project if diff_lines.is_a?(Gitlab::Diff::File) @diff_file = diff_lines @@ -66,7 +67,7 @@ module Gitlab end def inline_diffs - @inline_diffs ||= InlineDiff.for_lines(@raw_lines) + @inline_diffs ||= InlineDiff.for_lines(@raw_lines, project: project) end def old_lines diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 90cb9c8638a..7932cd2a837 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -8,6 +8,7 @@ module Gitlab EXPIRATION = 1.week VERSION = 1 + NEXT_VERSION = 2 delegate :diffable, to: :@diff_collection delegate :diff_options, to: :@diff_collection @@ -69,12 +70,20 @@ module Gitlab def key strong_memoize(:redis_key) do - ['highlighted-diff-files', diffable.cache_key, VERSION, diff_options].join(":") + ['highlighted-diff-files', diffable.cache_key, version, diff_options].join(":") end end private + def version + if Feature.enabled?(:improved_merge_diff_highlighting, diffable.project, default_enabled: :yaml) + NEXT_VERSION + else + VERSION + end + end + def set_highlighted_diff_lines(diff_file, content) diff_file.highlighted_diff_lines = content.map do |line| Gitlab::Diff::Line.safe_init_from_hash(line) diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 5815d1bae4a..cf769262958 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -27,28 +27,19 @@ module Gitlab @offset = offset end - def inline_diffs + def inline_diffs(project: nil) # Skip inline diff if empty line was replaced with content return if old_line == "" - lcp = longest_common_prefix(old_line, new_line) - lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) - - lcp += offset - old_length = old_line.length + offset - new_length = new_line.length + offset - - old_diff_range = lcp..(old_length - lcs - 1) - new_diff_range = lcp..(new_length - lcs - 1) - - old_diffs = [old_diff_range] if old_diff_range.begin <= old_diff_range.end - new_diffs = [new_diff_range] if new_diff_range.begin <= new_diff_range.end - - [old_diffs, new_diffs] + if Feature.enabled?(:improved_merge_diff_highlighting, project, default_enabled: :yaml) + CharDiff.new(old_line, new_line).changed_ranges(offset: offset) + else + deprecated_diff + end end class << self - def for_lines(lines) + def for_lines(lines, project: nil) changed_line_pairs = find_changed_line_pairs(lines) inline_diffs = [] @@ -57,7 +48,7 @@ module Gitlab old_line = lines[old_index] new_line = lines[new_index] - old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs + old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs(project: project) inline_diffs[old_index] = old_diffs inline_diffs[new_index] = new_diffs @@ -97,6 +88,24 @@ module Gitlab private + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/299884 + def deprecated_diff + lcp = longest_common_prefix(old_line, new_line) + lcs = longest_common_suffix(old_line[lcp..-1], new_line[lcp..-1]) + + lcp += offset + old_length = old_line.length + offset + new_length = new_line.length + offset + + old_diff_range = lcp..(old_length - lcs - 1) + new_diff_range = lcp..(new_length - lcs - 1) + + old_diffs = [old_diff_range] if old_diff_range.begin <= old_diff_range.end + new_diffs = [new_diff_range] if new_diff_range.begin <= new_diff_range.end + + [old_diffs, new_diffs] + end + def longest_common_prefix(a, b) # rubocop:disable Naming/UncommunicativeMethodParamName max_length = [a.length, b.length].max diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index f66e8a8794f..d1dd616385d 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -35,7 +35,11 @@ module Gitlab raise ProjectNotFound if project.nil? create_issue! - send_thank_you_email! if from_address + + if from_address + add_email_participant + send_thank_you_email! + end end def metrics_params @@ -146,6 +150,10 @@ module Gitlab def author User.support_bot end + + def add_email_participant + @issue.issue_email_participants.create(email: from_address) + end end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index cab21d875ab..e6f71e3ad3c 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -63,6 +63,16 @@ module Gitlab ActionController::Base.helpers.content_tag('gl-emoji', emoji_info['moji'], options) end + def custom_emoji_tag(name, image_source) + data = { + name: name + } + + ActionController::Base.helpers.content_tag('gl-emoji', title: name, data: data) do + emoji_image_tag(name, image_source).html_safe + end + end + private def emoji_unicode_versions_by_name diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 196203211ed..423f238a0a2 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -6,6 +6,7 @@ # Experiment options: # - tracking_category (optional, used to set the category when tracking an experiment event) # - use_backwards_compatible_subject_index (optional, set this to true if you need backwards compatibility -- you likely do not need this, see note in the next paragraph.) +# - rollout_strategy: default is `:cookie` based rollout. We may also set it to `:user` based rollout # # Using the backwards-compatible subject index (use_backwards_compatible_subject_index option): # This option was added when [the calculation of experimentation_subject_index was changed](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45733/diffs#41af4a6fa5a10c7068559ce21c5188483751d934_157_173). It is not intended to be used by new experiments, it exists merely for the segmentation integrity of in-flight experiments at the time the change was deployed. That is, we want users who were assigned to the "experimental" group or the "control" group before the change to still be in those same groups after the change. See [the original issue](https://gitlab.com/gitlab-org/gitlab/-/issues/270858) and [this related comment](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/48110#note_458223745) for more information. @@ -33,10 +34,6 @@ module Gitlab module Experimentation EXPERIMENTS = { - onboarding_issues: { - tracking_category: 'Growth::Conversion::Experiment::OnboardingIssues', - use_backwards_compatible_subject_index: true - }, ci_notification_dot: { tracking_category: 'Growth::Expansion::Experiment::CiNotificationDot', use_backwards_compatible_subject_index: true @@ -69,13 +66,6 @@ module Gitlab tracking_category: 'Growth::Conversion::Experiment::GroupOnlyTrials', use_backwards_compatible_subject_index: true }, - default_to_issues_board: { - tracking_category: 'Growth::Conversion::Experiment::DefaultToIssuesBoard', - use_backwards_compatible_subject_index: true - }, - jobs_empty_state: { - tracking_category: 'Growth::Activation::Experiment::JobsEmptyState' - }, remove_known_trial_form_fields: { tracking_category: 'Growth::Conversion::Experiment::RemoveKnownTrialFormFields' }, @@ -92,19 +82,27 @@ module Gitlab tracking_category: 'Growth::Conversion::Experiment::TrialDuringSignup' }, ci_syntax_templates: { - tracking_category: 'Growth::Activation::Experiment::CiSyntaxTemplates' - }, - pipelines_empty_state: { - tracking_category: 'Growth::Activation::Experiment::PipelinesEmptyState' + tracking_category: 'Growth::Activation::Experiment::CiSyntaxTemplates', + rollout_strategy: :user }, invite_members_new_dropdown: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' }, show_trial_status_in_sidebar: { - tracking_category: 'Growth::Conversion::Experiment::ShowTrialStatusInSidebar' + tracking_category: 'Growth::Conversion::Experiment::ShowTrialStatusInSidebar', + rollout_strategy: :group }, trial_onboarding_issues: { tracking_category: 'Growth::Conversion::Experiment::TrialOnboardingIssues' + }, + learn_gitlab_a: { + tracking_category: 'Growth::Conversion::Experiment::LearnGitLabA' + }, + learn_gitlab_b: { + tracking_category: 'Growth::Activation::Experiment::LearnGitLabB' + }, + in_product_marketing_emails: { + tracking_category: 'Growth::Activation::Experiment::InProductMarketingEmails' } }.freeze @@ -126,12 +124,44 @@ module Gitlab return false if subject.blank? return false unless active?(experiment_key) + log_invalid_rollout(experiment_key, subject) + experiment = get_experiment(experiment_key) return false unless experiment experiment.enabled_for_index?(index_for_subject(experiment, subject)) end + def rollout_strategy(experiment_key) + experiment = get_experiment(experiment_key) + return unless experiment + + experiment.rollout_strategy + end + + def log_invalid_rollout(experiment_key, subject) + return if valid_subject_for_rollout_strategy?(experiment_key, subject) + + logger = Gitlab::ExperimentationLogger.build + logger.warn message: 'Subject must conform to the rollout strategy', + experiment_key: experiment_key, + subject: subject.class.to_s, + rollout_strategy: rollout_strategy(experiment_key) + end + + def valid_subject_for_rollout_strategy?(experiment_key, subject) + case rollout_strategy(experiment_key) + when :user + subject.is_a?(User) + when :group + subject.is_a?(Group) + when :cookie + subject.nil? || subject.is_a?(String) + else + false + end + end + private def index_for_subject(experiment, subject) diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index e43f3c8c007..2b38b12c914 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -40,6 +40,8 @@ module Gitlab return true if forced_enabled?(experiment_key) return false if dnt_enabled? + Experimentation.log_invalid_rollout(experiment_key, subject) + subject ||= fallback_experimentation_subject_index(experiment_key) Experimentation.in_experiment_group?(experiment_key, subject: subject) @@ -65,7 +67,9 @@ module Gitlab return if dnt_enabled? return unless Experimentation.active?(experiment_key) && current_user - ::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: current_user), current_user, context) + subject = Experimentation.rollout_strategy(experiment_key) == :cookie ? nil : current_user + + ::Experiment.add_user(experiment_key, tracking_group(experiment_key, nil, subject: subject), current_user, context) end def record_experiment_conversion_event(experiment_key) @@ -136,7 +140,7 @@ module Gitlab cookies[:force_experiment].to_s.split(',').any? { |experiment| experiment.strip == experiment_key.to_s } end - def tracking_label(subject) + def tracking_label(subject = nil) return experimentation_subject_id if subject.blank? if subject.respond_to?(:to_global_id) diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb index 36cd673a38f..17dda45f5b7 100644 --- a/lib/gitlab/experimentation/experiment.rb +++ b/lib/gitlab/experimentation/experiment.rb @@ -5,12 +5,13 @@ module Gitlab class Experiment FEATURE_FLAG_SUFFIX = "_experiment_percentage" - attr_reader :key, :tracking_category, :use_backwards_compatible_subject_index + attr_reader :key, :tracking_category, :use_backwards_compatible_subject_index, :rollout_strategy def initialize(key, **params) @key = key @tracking_category = params[:tracking_category] @use_backwards_compatible_subject_index = params[:use_backwards_compatible_subject_index] + @rollout_strategy = params[:rollout_strategy] || :cookie end def active? diff --git a/lib/gitlab/experimentation_logger.rb b/lib/gitlab/experimentation_logger.rb new file mode 100644 index 00000000000..ba1b60d6b4c --- /dev/null +++ b/lib/gitlab/experimentation_logger.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + class ExperimentationLogger < ::Gitlab::JsonLogger + def self.file_name_noext + 'experimentation_json' + end + end +end diff --git a/lib/gitlab/faraday.rb b/lib/gitlab/faraday.rb deleted file mode 100644 index f92392ec1a9..00000000000 --- a/lib/gitlab/faraday.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Faraday - ::Faraday::Request.register_middleware(gitlab_error_callback: -> { ::Gitlab::Faraday::ErrorCallback }) - end -end diff --git a/lib/gitlab/file_type_detection.rb b/lib/gitlab/file_type_detection.rb index 38ccd2c38a9..ed25310b5cf 100644 --- a/lib/gitlab/file_type_detection.rb +++ b/lib/gitlab/file_type_detection.rb @@ -19,7 +19,7 @@ # `Content-Type` and `Content-Disposition` to the one we get from the detection. module Gitlab module FileTypeDetection - SAFE_IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico].freeze + SAFE_IMAGE_EXT = %w[png jpg jpeg gif bmp tiff ico webp].freeze SAFE_IMAGE_FOR_SCALING_EXT = %w[png jpg jpeg].freeze PDF_EXT = 'pdf' diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 0bc7ecccf5e..35c3dc5b0b3 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -16,7 +16,7 @@ module Gitlab SERIALIZE_KEYS = [ :id, :message, :parent_ids, :authored_date, :author_name, :author_email, - :committed_date, :committer_name, :committer_email + :committed_date, :committer_name, :committer_email, :trailers ].freeze attr_accessor(*SERIALIZE_KEYS) @@ -389,6 +389,7 @@ module Gitlab @committer_name = commit.committer.name.dup @committer_email = commit.committer.email.dup @parent_ids = Array(commit.parent_ids) + @trailers = Hash[commit.trailers.map { |t| [t.key, t.value] }] end # Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 209917073c7..53df0b7b389 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -244,6 +244,8 @@ module Gitlab def prune_diff_if_eligible if too_large? + ::Gitlab::Metrics.add_event(:patch_hard_limit_bytes_hit) + too_large! elsif collapsed? collapse! diff --git a/lib/gitlab/git/push.rb b/lib/gitlab/git/push.rb index b6577ba17f1..3d533a5185f 100644 --- a/lib/gitlab/git/push.rb +++ b/lib/gitlab/git/push.rb @@ -33,7 +33,9 @@ module Gitlab end def force_push? - Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + strong_memoize(:force_push) do + Gitlab::Checks::ForcePush.force_push?(@project, @oldrev, @newrev) + end end def branch_push? diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb index 0eff35ab1c4..0607b151de2 100644 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ b/lib/gitlab/git/rugged_impl/commit.rb @@ -103,6 +103,7 @@ module Gitlab @committer_name = committer[:name] @committer_email = committer[:email] @parent_ids = commit.parents.map(&:oid) + @trailers = Hash[commit.trailers] end end end diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 11919be594d..55ff3c6caf1 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -151,6 +151,8 @@ module Gitlab end def gitaly_find_page(title:, version: nil, dir: nil) + return unless title.present? + wiki_page, version = gitaly_wiki_client.find_page(title: title, version: version, dir: dir) return unless wiki_page diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index e0b145f69aa..c5ca46827cb 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -77,7 +77,6 @@ module Gitlab check_authentication_abilities! check_command_disabled! check_command_existence! - check_otp_session! custom_action = check_custom_action return custom_action if custom_action @@ -255,31 +254,6 @@ module Gitlab end end - def check_otp_session! - return unless ssh? - return if !key? || deploy_key? - return unless Feature.enabled?(:two_factor_for_cli) - return unless user.two_factor_enabled? - - if ::Gitlab::Auth::Otp::SessionEnforcer.new(actor).access_restricted? - message = "OTP verification is required to access the repository.\n\n"\ - " Use: #{build_ssh_otp_verify_command}" - - raise ForbiddenError, message - end - end - - def build_ssh_otp_verify_command - user = "#{Gitlab.config.gitlab_shell.ssh_user}@" unless Gitlab.config.gitlab_shell.ssh_user.empty? - user_host = "#{user}#{Gitlab.config.gitlab_shell.ssh_host}" - - if Gitlab.config.gitlab_shell.ssh_port != 22 - "ssh #{user_host} -p #{Gitlab.config.gitlab_shell.ssh_port} 2fa_verify" - else - "ssh #{user_host} 2fa_verify" - end - end - def check_db_accessibility! return unless receive_pack? @@ -345,10 +319,8 @@ module Gitlab end def check_change_access! - return if deploy_key? && !deploy_keys_on_protected_branches_enabled? - if changes == ANY - can_push = (deploy_key? && deploy_keys_on_protected_branches_enabled?) || + can_push = deploy_key? || user_can_push? || project&.any_branch_allows_collaboration?(user_access.user) @@ -479,7 +451,7 @@ module Gitlab CiAccess.new elsif user && request_from_ci_build? BuildAccess.new(user, container: container) - elsif deploy_key? && deploy_keys_on_protected_branches_enabled? + elsif deploy_key? DeployKeyAccess.new(deploy_key, container: container) else UserAccess.new(user, container: container) @@ -558,10 +530,6 @@ module Gitlab def size_checker container.repository_size_checker end - - def deploy_keys_on_protected_branches_enabled? - Feature.enabled?(:deploy_keys_on_protected_branches, project) - end end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 31734abe77f..3c7fa88977e 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -203,7 +203,7 @@ module Gitlab def self.authorization_token(storage) token = token(storage).to_s issued_at = real_time.to_i.to_s - hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, token, issued_at) + hmac = OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('SHA256'), token, issued_at) "v2.#{hmac}.#{issued_at}" end @@ -226,6 +226,7 @@ module Gitlab metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil) metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil) metadata.merge!(Feature::Gitaly.server_feature_flags) + metadata.merge!(route_to_primary) deadline_info = request_deadline(timeout) metadata.merge!(deadline_info.slice(:deadline_type)) @@ -233,6 +234,26 @@ module Gitlab { metadata: metadata, deadline: deadline_info[:deadline] } end + # Gitlab::Git::HookEnv will set the :gitlab_git_env variable in case we're + # running in the context of a Gitaly hook call, which may make use of + # quarantined object directories. We thus need to pass along the path of + # the quarantined object directory to Gitaly, otherwise it won't be able to + # find these quarantined objects. Given that the quarantine directory is + # generated with a random name, they'll have different names when multiple + # Gitaly nodes take part in a single transaction. As a result, we are + # forced to route all requests to the primary node which has injected the + # quarantine object directory to us. + def self.route_to_primary + return {} unless Gitlab::SafeRequestStore.active? + + return {} unless Gitlab::SafeRequestStore[:gitlab_git_env] + + return {} if Gitlab::SafeRequestStore[:gitlab_git_env].empty? + + { 'gitaly-route-repository-accessor-policy' => 'primary-only' } + end + private_class_method :route_to_primary + def self.request_deadline(timeout) # timeout being 0 means the request is allowed to run indefinitely. # We can't allow that inside a request, but this won't count towards Gitaly diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index ea940150941..ef5221a8042 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -335,7 +335,8 @@ module Gitlab all: !!options[:all], first_parent: !!options[:first_parent], global_options: parse_global_options!(options), - disable_walk: true # This option is deprecated. The 'walk' implementation is being removed. + disable_walk: true, # This option is deprecated. The 'walk' implementation is being removed. + trailers: options[:trailers] ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index 6f08dcc69b6..fc40c23611a 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -67,7 +67,8 @@ module Gitlab source_branch: encode_binary(source_branch), target_branch: encode_binary(target_branch), commit_message: encode_binary(resolution.commit_message), - user: Gitlab::Git::User.from_gitlab(resolution.user).to_gitaly + user: Gitlab::Git::User.from_gitlab(resolution.user).to_gitaly, + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 4850d646de4..6f302b2c4e7 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -32,7 +32,8 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, tag_name: encode_binary(tag_name), target_revision: encode_binary(target), - message: encode_binary(message.to_s) + message: encode_binary(message.to_s), + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout) @@ -111,7 +112,8 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, message: encode_binary(message), first_parent_ref: encode_binary(first_parent_ref), - allow_conflicts: allow_conflicts + allow_conflicts: allow_conflicts, + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) response = GitalyClient.call(@repository.storage, :operation_service, @@ -140,7 +142,8 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, commit_id: source_sha, branch: encode_binary(target_branch), - message: encode_binary(message) + message: encode_binary(message), + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) ) @@ -234,7 +237,8 @@ module Gitlab branch_sha: branch_sha, remote_repository: remote_repository.gitaly_repository, remote_branch: encode_binary(remote_branch), - git_push_options: push_options + git_push_options: push_options, + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) ) ) @@ -255,7 +259,7 @@ module Gitlab request_enum.close end - def user_squash(user, squash_id, start_sha, end_sha, author, message) + def user_squash(user, squash_id, start_sha, end_sha, author, message, time = Time.now.utc) request = Gitaly::UserSquashRequest.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, @@ -263,7 +267,8 @@ module Gitlab start_sha: start_sha, end_sha: end_sha, author: Gitlab::Git::User.from_gitlab(author).to_gitaly, - commit_message: encode_binary(message) + commit_message: encode_binary(message), + timestamp: Google::Protobuf::Timestamp.new(seconds: time.to_i) ) response = GitalyClient.call( @@ -288,7 +293,8 @@ module Gitlab commit_sha: commit_sha, branch: encode_binary(branch), submodule: encode_binary(submodule), - commit_message: encode_binary(message) + commit_message: encode_binary(message), + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) response = GitalyClient.call( @@ -357,7 +363,8 @@ module Gitlab header = Gitaly::UserApplyPatchRequest::Header.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - target_branch: encode_binary(branch_name) + target_branch: encode_binary(branch_name), + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) reader = binary_io(patches) @@ -446,7 +453,8 @@ module Gitlab start_branch_name: encode_binary(start_branch_name), start_repository: start_repository.gitaly_repository, force: force, - start_sha: encode_binary(start_sha) + start_sha: encode_binary(start_sha), + timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) end # rubocop:enable Metrics/ParameterLists diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 43848772947..5e50ac72965 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -23,7 +23,6 @@ module Gitlab MUTEX = Mutex.new - DISK_ACCESS_DENIED_FLAG = :deny_disk_access ALLOW_KEY = :allow_disk_access # If your code needs this method then your code needs to be fixed. @@ -34,7 +33,7 @@ module Gitlab def self.disk_access_denied? return false if rugged_enabled? - !temporarily_allowed?(ALLOW_KEY) && Feature::Gitaly.enabled?(DISK_ACCESS_DENIED_FLAG) + !temporarily_allowed?(ALLOW_KEY) rescue false # Err on the side of caution, don't break gitlab for people end @@ -62,7 +61,7 @@ module Gitlab def legacy_disk_path if self.class.disk_access_denied? - raise DirectPathAccessError, "git disk access denied via the gitaly_#{DISK_ACCESS_DENIED_FLAG} feature" + raise DirectPathAccessError, "git disk access denied" end @legacy_disk_path diff --git a/lib/gitlab/global_id.rb b/lib/gitlab/global_id.rb index e8a6006dce1..7e9412236cf 100644 --- a/lib/gitlab/global_id.rb +++ b/lib/gitlab/global_id.rb @@ -19,8 +19,8 @@ module Gitlab value when URI::GID GlobalID.new(value) - when Integer - raise CoerceError, 'Cannot coerce Integer' unless model_name.present? + when Integer, String + raise CoerceError, "Cannot coerce #{value.class}" unless model_name.present? GlobalID.new(::Gitlab::GlobalId.build(model_name: model_name, id: value)) else diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 0ba535b500e..3dd317c5a64 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -13,7 +13,6 @@ module Gitlab gon.asset_host = ActionController::Base.asset_host gon.webpack_public_path = webpack_public_path gon.relative_url_root = Gitlab.config.gitlab.relative_url_root - gon.shortcuts_path = Gitlab::Routing.url_helpers.help_page_path('shortcuts') gon.user_color_scheme = Gitlab::ColorSchemes.for_user(current_user).css_class if Gitlab.config.sentry.enabled @@ -33,6 +32,7 @@ module Gitlab gon.suggested_label_colors = LabelsHelper.suggested_colors gon.first_day_of_week = current_user&.first_day_of_week || Gitlab::CurrentSettings.first_day_of_week gon.ee = Gitlab.ee? + gon.dot_com = Gitlab.com? if current_user gon.current_user_id = current_user.id @@ -43,11 +43,9 @@ module Gitlab # Initialize gon.features with any flags that should be # made globally available to the frontend - push_frontend_feature_flag(:webperf_experiment, default_enabled: false) push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) push_frontend_feature_flag(:usage_data_api, default_enabled: true) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) - push_frontend_feature_flag(:gl_tooltips, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/docs/templates/default.md.haml b/lib/gitlab/graphql/docs/templates/default.md.haml index 8f5a1788fa5..9dfb9b090a8 100644 --- a/lib/gitlab/graphql/docs/templates/default.md.haml +++ b/lib/gitlab/graphql/docs/templates/default.md.haml @@ -16,6 +16,8 @@ Fields that are deprecated are marked with **{warning-solid}**. Items (fields, enums, etc) that have been removed according to our [deprecation process](../index.md#deprecation-process) can be found in [Removed Items](../removed_items.md). + + <!-- vale gitlab.Spelling = NO --> \ :plain diff --git a/lib/gitlab/graphql/pagination/connections.rb b/lib/gitlab/graphql/pagination/connections.rb index 54a84be4274..965c01dd02f 100644 --- a/lib/gitlab/graphql/pagination/connections.rb +++ b/lib/gitlab/graphql/pagination/connections.rb @@ -6,6 +6,10 @@ module Gitlab module Connections def self.use(schema) schema.connections.add( + ::Gitlab::Graphql::Pagination::OffsetPaginatedRelation, + ::Gitlab::Graphql::Pagination::OffsetActiveRecordRelationConnection) + + schema.connections.add( ActiveRecord::Relation, Gitlab::Graphql::Pagination::Keyset::Connection) diff --git a/lib/gitlab/graphql/pagination/offset_paginated_relation.rb b/lib/gitlab/graphql/pagination/offset_paginated_relation.rb new file mode 100644 index 00000000000..8a8c6e5db50 --- /dev/null +++ b/lib/gitlab/graphql/pagination/offset_paginated_relation.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# Marker class to enable us to choose the correct +# connection type during resolution +module Gitlab + module Graphql + module Pagination + class OffsetPaginatedRelation < SimpleDelegator + end + end + end +end diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb index de971743490..fcf293fb13e 100644 --- a/lib/gitlab/graphql/queries.rb +++ b/lib/gitlab/graphql/queries.rb @@ -145,6 +145,20 @@ module Gitlab return redacted if printer.fields_printed > 0 end + def complexity(schema) + # See BaseResolver::resolver_complexity + # we want to see the max possible complexity. + fake_args = Struct + .new(:if, :keyword_arguments) + .new(nil, { sort: true, search: true }) + + query = GraphQL::Query.new(schema, text) + # We have no arguments, so fake them. + query.define_singleton_method(:arguments_for) { |_x, _y| fake_args } + + GraphQL::Analysis::AST.analyze_query(query, [GraphQL::Analysis::AST::QueryComplexity]).first + end + def query return @query if defined?(@query) diff --git a/lib/gitlab/health_checks/base_abstract_check.rb b/lib/gitlab/health_checks/base_abstract_check.rb index 7e1c5331b07..2eef359cc6e 100644 --- a/lib/gitlab/health_checks/base_abstract_check.rb +++ b/lib/gitlab/health_checks/base_abstract_check.rb @@ -11,6 +11,10 @@ module Gitlab name.sub(/_check$/, '').capitalize end + def available? + true + end + def readiness raise NotImplementedError end diff --git a/lib/gitlab/health_checks/master_check.rb b/lib/gitlab/health_checks/master_check.rb index 057bce84ddd..b2c3695e6d9 100644 --- a/lib/gitlab/health_checks/master_check.rb +++ b/lib/gitlab/health_checks/master_check.rb @@ -8,7 +8,16 @@ module Gitlab extend SimpleAbstractCheck class << self + extend ::Gitlab::Utils::Override + + override :available? + def available? + Gitlab::Runtime.puma_in_clustered_mode? + end + def register_master + return unless available? + # when we fork, we pass the read pipe to child # child can then react on whether the other end # of pipe is still available @@ -16,11 +25,15 @@ module Gitlab end def finish_master + return unless available? + close_read close_write end def register_worker + return unless available? + # fork needs to close the pipe close_write end diff --git a/lib/gitlab/health_checks/probes/collection.rb b/lib/gitlab/health_checks/probes/collection.rb index 08b6d82291e..b34e4273d85 100644 --- a/lib/gitlab/health_checks/probes/collection.rb +++ b/lib/gitlab/health_checks/probes/collection.rb @@ -48,6 +48,7 @@ module Gitlab def probe_readiness checks + .select(&:available?) .flat_map(&:readiness) .compact .group_by(&:name) diff --git a/lib/gitlab/hook_data/base_builder.rb b/lib/gitlab/hook_data/base_builder.rb index 434d30d9717..e5bae61ae4e 100644 --- a/lib/gitlab/hook_data/base_builder.rb +++ b/lib/gitlab/hook_data/base_builder.rb @@ -21,6 +21,12 @@ module Gitlab private + def event_data(event) + event_name = "#{object.class.name.downcase}_#{event}" + + { event_name: event_name } + end + def timestamps_data { created_at: object.created_at&.xmlschema, diff --git a/lib/gitlab/hook_data/group_builder.rb b/lib/gitlab/hook_data/group_builder.rb new file mode 100644 index 00000000000..5f76144eb83 --- /dev/null +++ b/lib/gitlab/hook_data/group_builder.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class GroupBuilder < BaseBuilder + alias_method :group, :object + + # Sample data + # { + # :created_at=>"2021-01-20T09:40:12Z", + # :updated_at=>"2021-01-20T09:40:12Z", + # :event_name=>"group_rename", + # :name=>"group1", + # :path=>"group1", + # :full_path=>"group1", + # :group_id=>1, + # :old_path=>"old-path", + # :old_full_path=>"old-path" + # } + + def build(event) + [ + timestamps_data, + event_data(event), + group_data, + event_specific_group_data(event) + ].reduce(:merge) + end + + private + + def group_data + { + name: group.name, + path: group.path, + full_path: group.full_path, + group_id: group.id + } + end + + def event_specific_group_data(event) + return {} unless event == :rename + + { + old_path: group.path_before_last_save, + old_full_path: group.full_path_before_last_save + } + end + end + end +end diff --git a/lib/gitlab/hook_data/subgroup_builder.rb b/lib/gitlab/hook_data/subgroup_builder.rb new file mode 100644 index 00000000000..a620219675a --- /dev/null +++ b/lib/gitlab/hook_data/subgroup_builder.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class SubgroupBuilder < GroupBuilder + # Sample data + # { + # :created_at=>"2021-01-20T09:40:12Z", + # :updated_at=>"2021-01-20T09:40:12Z", + # :event_name=>"subgroup_create", + # :name=>"subgroup1", + # :path=>"subgroup1", + # :full_path=>"group1/subgroup1", + # :group_id=>10, + # :parent_group_id=>7, + # :parent_name=>group1, + # :parent_path=>group1, + # :parent_full_path=>group1 + # } + + private + + def event_data(event) + event_name = case event + when :create + 'subgroup_create' + when :destroy + 'subgroup_destroy' + end + + { event_name: event_name } + end + + def group_data + parent = group.parent + + super.merge( + parent_group_id: parent.id, + parent_name: parent.name, + parent_path: parent.path, + parent_full_path: parent.full_path + ) + end + + def event_specific_group_data(event) + {} + end + end + end +end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 921072a4970..c4867746b0f 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -103,3 +103,7 @@ module Gitlab end Gitlab::ImportExport.prepend_if_ee('EE::Gitlab::ImportExport') + +# The methods in `Gitlab::ImportExport::GroupHelper` should be available as both +# instance and class methods. +Gitlab::ImportExport.extend_if_ee('Gitlab::ImportExport::GroupHelper') diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 219821a7150..37f1bdc3009 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -1,24 +1,16 @@ # frozen_string_literal: true -require 'zlib' - module Gitlab module ImportExport class DecompressedArchiveSizeValidator include Gitlab::Utils::StrongMemoize DEFAULT_MAX_BYTES = 10.gigabytes.freeze - CHUNK_SIZE = 4096.freeze - - attr_reader :error + TIMEOUT_LIMIT = 60.seconds def initialize(archive_path:, max_bytes: self.class.max_bytes) @archive_path = archive_path @max_bytes = max_bytes - @bytes_read = 0 - @total_reads = 0 - @denominator = 5 - @error = nil end def valid? @@ -31,59 +23,62 @@ module Gitlab DEFAULT_MAX_BYTES end - def archive_file - @archive_file ||= File.open(@archive_path) - end - private def validate - until archive_file.eof? - compressed_chunk = archive_file.read(CHUNK_SIZE) + pgrp = nil + valid_archive = true - inflate_stream.inflate(compressed_chunk) do |chunk| - @bytes_read += chunk.size - @total_reads += 1 - end + Timeout.timeout(TIMEOUT_LIMIT) do + stdin, stdout, stderr, wait_thr = Open3.popen3(command, pgroup: true) + stdin.close + pgrp = Process.getpgid(wait_thr[:pid]) + status = wait_thr.value - # Start garbage collection every 5 reads in order - # to prevent memory bloat during archive decompression - GC.start if gc_start? + if status.success? + result = stdout.readline - if @bytes_read > @max_bytes - @error = error_message + if result.to_i > @max_bytes + valid_archive = false - return false + log_error('Decompressed archive size limit reached') + end + else + valid_archive = false + + log_error(stderr.readline) end + + ensure + stdout.close + stderr.close end - true - rescue => e - @error = error_message + valid_archive + rescue Timeout::Error + log_error('Timeout reached during archive decompression') - Gitlab::ErrorTracking.track_exception(e) - - Gitlab::Import::Logger.info( - message: @error, - error: e.message - ) + Process.kill(-1, pgrp) if pgrp false - ensure - inflate_stream.close - archive_file.close - end + rescue => e + log_error(e.message) - def inflate_stream - @inflate_stream ||= Zlib::Inflate.new(Zlib::MAX_WBITS + 32) + Process.kill(-1, pgrp) if pgrp + + false end - def gc_start? - @total_reads % @denominator == 0 + def command + "gzip -dc #{@archive_path} | wc -c" end - def error_message - _('Decompressed archive size validation failed.') + def log_error(error) + Gitlab::Import::Logger.info( + message: error, + import_upload_archive_path: @archive_path, + import_upload_archive_size: File.size(@archive_path) + ) end end end diff --git a/lib/gitlab/import_export/design_repo_restorer.rb b/lib/gitlab/import_export/design_repo_restorer.rb index a702c58a7c2..e093b4b0697 100644 --- a/lib/gitlab/import_export/design_repo_restorer.rb +++ b/lib/gitlab/import_export/design_repo_restorer.rb @@ -3,10 +3,11 @@ module Gitlab module ImportExport class DesignRepoRestorer < RepoRestorer - def initialize(project:, shared:, path_to_bundle:) - super(project: project, shared: shared, path_to_bundle: path_to_bundle) + extend ::Gitlab::Utils::Override - @repository = project.design_repository + override :repository + def repository + @repository ||= importable.design_repository end # `restore` method is handled in super class diff --git a/lib/gitlab/import_export/design_repo_saver.rb b/lib/gitlab/import_export/design_repo_saver.rb index db9ebee6a13..b400aedc205 100644 --- a/lib/gitlab/import_export/design_repo_saver.rb +++ b/lib/gitlab/import_export/design_repo_saver.rb @@ -3,16 +3,18 @@ module Gitlab module ImportExport class DesignRepoSaver < RepoSaver - def save - @repository = project.design_repository + extend ::Gitlab::Utils::Override - super + override :repository + def repository + @repository ||= exportable.design_repository end private - def bundle_full_path - File.join(shared.export_path, ::Gitlab::ImportExport.design_repo_bundle_filename) + override :bundle_filename + def bundle_filename + ::Gitlab::ImportExport.design_repo_bundle_filename end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 5a6f6e017d2..51d58aae54f 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -87,7 +87,7 @@ module Gitlab end def validate_decompressed_archive_size - raise ImporterError.new(size_validator.error) unless size_validator.valid? + raise ImporterError.new(_('Decompressed archive size validation failed.')) unless size_validator.valid? end def size_validator diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index dfe27118d66..925ab6680ba 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -6,7 +6,7 @@ module Gitlab class TreeRestorer include Gitlab::Utils::StrongMemoize - attr_reader :user, :shared + attr_reader :user, :shared, :groups_mapping def initialize(user:, shared:, group:) @user = user diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 789249c7d91..390909efe36 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -75,19 +75,19 @@ module Gitlab def repo_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: repo_path, shared: shared, - project: project) + importable: project) end def wiki_restorer Gitlab::ImportExport::RepoRestorer.new(path_to_bundle: wiki_repo_path, shared: shared, - project: ProjectWiki.new(project)) + importable: ProjectWiki.new(project)) end def design_repo_restorer Gitlab::ImportExport::DesignRepoRestorer.new(path_to_bundle: design_repo_path, shared: shared, - project: project) + importable: project) end def uploads_restorer diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index f808e30bd6e..8af7b68d78e 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -5,10 +5,12 @@ module Gitlab class RepoRestorer include Gitlab::ImportExport::CommandLineUtil - def initialize(project:, shared:, path_to_bundle:) - @repository = project.repository + attr_reader :importable + + def initialize(importable:, shared:, path_to_bundle:) @path_to_bundle = path_to_bundle @shared = shared + @importable = importable end def restore @@ -17,14 +19,25 @@ module Gitlab ensure_repository_does_not_exist! repository.create_from_bundle(path_to_bundle) + update_importable_repository_info + + true rescue => e shared.error(e) false end + def repository + @repository ||= importable.repository + end + private - attr_accessor :repository, :path_to_bundle, :shared + attr_accessor :path_to_bundle, :shared + + def update_importable_repository_info + # No-op. Overridden in EE + end def ensure_repository_does_not_exist! if repository.exists? @@ -38,3 +51,5 @@ module Gitlab end end end + +Gitlab::ImportExport::RepoRestorer.prepend_if_ee('EE::Gitlab::ImportExport::RepoRestorer') diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 898cd7898ba..0fdd0722b65 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -5,12 +5,11 @@ module Gitlab class RepoSaver include Gitlab::ImportExport::CommandLineUtil - attr_reader :project, :repository, :shared + attr_reader :exportable, :shared - def initialize(project:, shared:) - @project = project + def initialize(exportable:, shared:) + @exportable = exportable @shared = shared - @repository = @project.repository end def save @@ -19,6 +18,10 @@ module Gitlab bundle_to_disk end + def repository + @repository ||= @exportable.repository + end + private def repository_exists? @@ -26,11 +29,16 @@ module Gitlab end def bundle_full_path - File.join(shared.export_path, ImportExport.project_bundle_filename) + File.join(shared.export_path, bundle_filename) + end + + def bundle_filename + ::Gitlab::ImportExport.project_bundle_filename end def bundle_to_disk - mkdir_p(shared.export_path) + mkdir_p(File.dirname(bundle_full_path)) + repository.bundle_to_disk(bundle_full_path) rescue => e shared.error(e) diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index 045ba2495bf..bb2bbda4bd6 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -31,7 +31,7 @@ module Gitlab @shared.error(e) false ensure - remove_base_tmp_dir + remove_archive_tmp_dir end private @@ -40,8 +40,8 @@ module Gitlab tar_czf(archive: archive_file, dir: @shared.export_path) end - def remove_base_tmp_dir - FileUtils.rm_rf(@shared.base_path) + def remove_archive_tmp_dir + FileUtils.rm_rf(@shared.archive_path) end def archive_file diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 93ae6f6b02a..4b1cf4915e4 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -3,18 +3,21 @@ module Gitlab module ImportExport class WikiRepoSaver < RepoSaver - def save - wiki = ProjectWiki.new(project) - @repository = wiki.repository + extend ::Gitlab::Utils::Override - super + override :repository + def repository + @repository ||= exportable.wiki.repository end private - def bundle_full_path - File.join(shared.export_path, ImportExport.wiki_repo_bundle_filename) + override :bundle_filename + def bundle_filename + ::Gitlab::ImportExport.wiki_repo_bundle_filename end end end end + +Gitlab::ImportExport::WikiRepoSaver.prepend_if_ee('EE::Gitlab::ImportExport::WikiRepoSaver') diff --git a/lib/gitlab/instrumentation/elasticsearch_transport.rb b/lib/gitlab/instrumentation/elasticsearch_transport.rb index 56179eda22d..4bef043ecb0 100644 --- a/lib/gitlab/instrumentation/elasticsearch_transport.rb +++ b/lib/gitlab/instrumentation/elasticsearch_transport.rb @@ -9,12 +9,17 @@ module Gitlab start = Time.now headers = (headers || {}) .reverse_merge({ 'X-Opaque-Id': Labkit::Correlation::CorrelationId.current_or_new_id }) - super + response = super ensure if ::Gitlab::SafeRequestStore.active? duration = (Time.now - start) ::Gitlab::Instrumentation::ElasticsearchTransport.increment_request_count + + if response&.body && response.body.is_a?(Hash) && response.body['timed_out'] + ::Gitlab::Instrumentation::ElasticsearchTransport.increment_timed_out_count + end + ::Gitlab::Instrumentation::ElasticsearchTransport.add_duration(duration) ::Gitlab::Instrumentation::ElasticsearchTransport.add_call_details(duration, method, path, params, body) end @@ -25,6 +30,7 @@ module Gitlab ELASTICSEARCH_REQUEST_COUNT = :elasticsearch_request_count ELASTICSEARCH_CALL_DURATION = :elasticsearch_call_duration ELASTICSEARCH_CALL_DETAILS = :elasticsearch_call_details + ELASTICSEARCH_TIMED_OUT_COUNT = :elasticsearch_timed_out_count def self.get_request_count ::Gitlab::SafeRequestStore[ELASTICSEARCH_REQUEST_COUNT] || 0 @@ -49,6 +55,15 @@ module Gitlab ::Gitlab::SafeRequestStore[ELASTICSEARCH_CALL_DURATION] += duration end + def self.increment_timed_out_count + ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] ||= 0 + ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] += 1 + end + + def self.get_timed_out_count + ::Gitlab::SafeRequestStore[ELASTICSEARCH_TIMED_OUT_COUNT] || 0 + end + def self.add_call_details(duration, method, path, params, body) return unless Gitlab::PerformanceBar.enabled_for_request? diff --git a/lib/gitlab/instrumentation/redis_cluster_validator.rb b/lib/gitlab/instrumentation/redis_cluster_validator.rb index 6800e5667f6..644a5fc4fff 100644 --- a/lib/gitlab/instrumentation/redis_cluster_validator.rb +++ b/lib/gitlab/instrumentation/redis_cluster_validator.rb @@ -61,7 +61,7 @@ module Gitlab key_slot(args.first) end - unless key_slots.uniq.length == 1 + if key_slots.uniq.many? # rubocop: disable CodeReuse/ActiveRecord raise CrossSlotError.new("Redis command #{command_name} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands") end end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 6b0f01757b7..61de6b02453 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -7,14 +7,33 @@ module Gitlab DURATION_PRECISION = 6 # microseconds def keys - @keys ||= [:gitaly_calls, - :gitaly_duration_s, - :rugged_calls, - :rugged_duration_s, - :elasticsearch_calls, - :elasticsearch_duration_s, - *::Gitlab::Instrumentation::Redis.known_payload_keys, - *::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS] + @keys ||= [ + :cpu_s, + :gitaly_calls, + :gitaly_duration_s, + :rugged_calls, + :rugged_duration_s, + :elasticsearch_calls, + :elasticsearch_duration_s, + :elasticsearch_timed_out_count, + *::Gitlab::Memory::Instrumentation::KEY_MAPPING.values, + *::Gitlab::Instrumentation::Redis.known_payload_keys, + *::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS, + *::Gitlab::Metrics::Subscribers::ExternalHttp::KNOWN_PAYLOAD_KEYS, + *::Gitlab::Metrics::Subscribers::RackAttack::PAYLOAD_KEYS + ] + end + + def init_instrumentation_data(request_ip: nil) + # Set `request_start_time` only if this is request + # This is done, as `request_start_time` imply `request_deadline` + if request_ip + Gitlab::RequestContext.instance.client_ip = request_ip + Gitlab::RequestContext.instance.request_start_time = Gitlab::Metrics::System.real_time + end + + Gitlab::RequestContext.instance.start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + Gitlab::RequestContext.instance.thread_memory_allocations = Gitlab::Memory::Instrumentation.start_thread_memory_allocations end def add_instrumentation_data(payload) @@ -24,6 +43,10 @@ module Gitlab instrument_elasticsearch(payload) instrument_throttle(payload) instrument_active_record(payload) + instrument_external_http(payload) + instrument_rack_attack(payload) + instrument_cpu(payload) + instrument_thread_memory_allocations(payload) end def instrument_gitaly(payload) @@ -57,6 +80,15 @@ module Gitlab payload[:elasticsearch_calls] = elasticsearch_calls payload[:elasticsearch_duration_s] = Gitlab::Instrumentation::ElasticsearchTransport.query_time + payload[:elasticsearch_timed_out_count] = Gitlab::Instrumentation::ElasticsearchTransport.get_timed_out_count + end + + def instrument_external_http(payload) + external_http_count = Gitlab::Metrics::Subscribers::ExternalHttp.request_count + + return if external_http_count == 0 + + payload.merge! Gitlab::Metrics::Subscribers::ExternalHttp.payload end def instrument_throttle(payload) @@ -70,6 +102,26 @@ module Gitlab payload.merge!(db_counters) end + def instrument_rack_attack(payload) + rack_attack_redis_count = ::Gitlab::Metrics::Subscribers::RackAttack.payload[:rack_attack_redis_count] + return if rack_attack_redis_count == 0 + + payload.merge!(::Gitlab::Metrics::Subscribers::RackAttack.payload) + end + + def instrument_cpu(payload) + cpu_s = ::Gitlab::Metrics::System.thread_cpu_duration( + ::Gitlab::RequestContext.instance.start_thread_cpu_time) + + payload[:cpu_s] = cpu_s.round(DURATION_PRECISION) if cpu_s + end + + def instrument_thread_memory_allocations(payload) + counters = ::Gitlab::Memory::Instrumentation.measure_thread_memory_allocations( + ::Gitlab::RequestContext.instance.thread_memory_allocations) + payload.merge!(counters) if counters + end + # Returns the queuing duration for a Sidekiq job in seconds, as a float, if the # `enqueued_at` field or `created_at` field is available. # @@ -96,7 +148,7 @@ module Gitlab # # @param [Time] start def self.elapsed_by_absolute_time(start) - (Time.now - start).to_f.round(6) + (Time.now - start).to_f.round(DURATION_PRECISION) end private_class_method :elapsed_by_absolute_time diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 08dde98e965..329c0f221b5 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -23,6 +23,12 @@ module Gitlab write_secret end + + def included_in_gitlab_com_rollout?(project) + return true unless ::Gitlab.com? + + Feature.enabled?(:kubernetes_agent_on_gitlab_com, project) + end end end end diff --git a/lib/gitlab/kroki.rb b/lib/gitlab/kroki.rb index 8c5652fb766..2948b6ebd5b 100644 --- a/lib/gitlab/kroki.rb +++ b/lib/gitlab/kroki.rb @@ -13,11 +13,8 @@ module Gitlab packetdiag rackdiag ].freeze - # Diagrams that require a companion container are disabled for now - DIAGRAMS_FORMATS = ::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES - .reject { |diagram_type| diagram_type == 'mermaid' || diagram_type == 'bpmn' || BLOCKDIAG_FORMATS.include?(diagram_type) } - DIAGRAMS_FORMATS_WO_PLANTUML = DIAGRAMS_FORMATS - .reject { |diagram_type| diagram_type == 'plantuml' } + DIAGRAMS_FORMATS = (::AsciidoctorExtensions::Kroki::SUPPORTED_DIAGRAM_NAMES - %w(mermaid)).freeze + DIAGRAMS_FORMATS_WO_PLANTUML = (DIAGRAMS_FORMATS - %w(plantuml)).freeze # Get the list of diagram formats that are currently enabled # @@ -28,10 +25,18 @@ module Gitlab # If PlantUML is enabled, PlantUML diagrams will be processed by the PlantUML server. # In other words, the PlantUML server has precedence over Kroki since both can process PlantUML diagrams. - if current_settings.plantuml_enabled - DIAGRAMS_FORMATS_WO_PLANTUML - else - DIAGRAMS_FORMATS + diagram_formats = if current_settings.plantuml_enabled + DIAGRAMS_FORMATS_WO_PLANTUML + else + DIAGRAMS_FORMATS + end + + # No additional diagram formats + return diagram_formats unless current_settings.kroki_formats.present? + + # Diagrams that require a companion container must be explicitly enabled from the settings + diagram_formats.select do |diagram_type| + current_settings.kroki_format_supported?(diagram_type) end end end diff --git a/lib/gitlab/kubernetes/helm/v2/certificate.rb b/lib/gitlab/kubernetes/helm/v2/certificate.rb index f603ff44ef3..17ea2eb5188 100644 --- a/lib/gitlab/kubernetes/helm/v2/certificate.rb +++ b/lib/gitlab/kubernetes/helm/v2/certificate.rb @@ -59,7 +59,7 @@ module Gitlab cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true)) end - cert.sign(signed_by&.key || key, OpenSSL::Digest::SHA256.new) + cert.sign(signed_by&.key || key, OpenSSL::Digest.new('SHA256')) new(key, cert) end diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index e6dd87a8bec..dce7cdb31a1 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -29,10 +29,6 @@ module Gitlab payload[:etag_route] = event.payload[:etag_route] if event.payload[:etag_route] payload[Labkit::Correlation::CorrelationId::LOG_KEY] = event.payload[Labkit::Correlation::CorrelationId::LOG_KEY] || Labkit::Correlation::CorrelationId.current_id - if cpu_s = Gitlab::Metrics::System.thread_cpu_duration(::Gitlab::RequestContext.instance.start_thread_cpu_time) - payload[:cpu_s] = cpu_s.round(2) - end - CLOUDFLARE_CUSTOM_HEADERS.each do |_, value| payload[value] = event.payload[value] if event.payload[value] end diff --git a/lib/gitlab/memory/instrumentation.rb b/lib/gitlab/memory/instrumentation.rb new file mode 100644 index 00000000000..76e84e54d3a --- /dev/null +++ b/lib/gitlab/memory/instrumentation.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +# This class uses a custom Ruby patch to allow +# a per-thread memory allocation tracking in a efficient manner +# +# This concept is currently tried to be upstreamed here: +# - https://github.com/ruby/ruby/pull/3978 +module Gitlab + module Memory + class Instrumentation + KEY_MAPPING = { + total_allocated_objects: :mem_objects, + total_malloc_bytes: :mem_bytes, + total_mallocs: :mem_mallocs + }.freeze + + MUTEX = Mutex.new + + def self.available? + Thread.respond_to?(:trace_memory_allocations=) && + Thread.current.respond_to?(:memory_allocations) + end + + # This method changes a global state + def self.ensure_feature_flag! + return unless available? + + enabled = Feature.enabled?(:trace_memory_allocations) + return if enabled == Thread.trace_memory_allocations + + MUTEX.synchronize do + # This enables or disables feature dynamically + # based on a feature flag + Thread.trace_memory_allocations = enabled + end + end + + def self.start_thread_memory_allocations + return unless available? + + ensure_feature_flag! + + # it will return `nil` if disabled + Thread.current.memory_allocations + end + + # This method returns a hash with the following keys: + # - mem_objects: a number of allocated heap slots (as reflected by GC) + # - mem_mallocs: a number of malloc calls + # - mem_bytes: a number of bytes allocated with a mallocs tied to heap slots + def self.measure_thread_memory_allocations(previous) + return unless available? + return unless previous + + current = Thread.current.memory_allocations + return unless current + + # calculate difference in a memory allocations + previous.to_h do |key, value| + [KEY_MAPPING.fetch(key), current[key].to_i - value] + end + end + + def self.with_memory_allocations + previous = self.start_thread_memory_allocations + yield + self.measure_thread_memory_allocations(previous) + end + end + end +end diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index b6a27d8556a..558454eaa1c 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -12,6 +12,10 @@ module Gitlab Gitlab::HealthChecks::Result.new( 'web_exporter', exporter.running) end + + def available? + true + end end attr_reader :running diff --git a/lib/gitlab/metrics/methods.rb b/lib/gitlab/metrics/methods.rb index 3100450bc00..8ddd76ad7ae 100644 --- a/lib/gitlab/metrics/methods.rb +++ b/lib/gitlab/metrics/methods.rb @@ -39,7 +39,7 @@ module Gitlab options.evaluate(&block) if disabled_by_feature(options) - synchronized_cache_fill(name) { NullMetric.instance } + synchronized_cache_fill(name) { ::Gitlab::Metrics::NullMetric.instance } else synchronized_cache_fill(name) { build_metric!(type, name, options) } end diff --git a/lib/gitlab/metrics/rack_middleware.rb b/lib/gitlab/metrics/rack_middleware.rb index a6884ea6983..f7e53bf545b 100644 --- a/lib/gitlab/metrics/rack_middleware.rb +++ b/lib/gitlab/metrics/rack_middleware.rb @@ -10,7 +10,7 @@ module Gitlab # env - A Hash containing Rack environment details. def call(env) - trans = WebTransaction.new(env) + trans = Gitlab::Metrics::WebTransaction.new(env) begin retval = trans.run { @app.call(env) } diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb new file mode 100644 index 00000000000..94c5d965200 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/external_http.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Subscribers + # Class for tracking the total time spent in external HTTP + # See more at https://gitlab.com/gitlab-org/labkit-ruby/-/blob/v0.14.0/lib/gitlab-labkit.rb#L18 + class ExternalHttp < ActiveSupport::Subscriber + attach_to :external_http + + DEFAULT_STATUS_CODE = 'undefined' + + DETAIL_STORE = :external_http_detail_store + COUNTER = :external_http_count + DURATION = :external_http_duration_s + + KNOWN_PAYLOAD_KEYS = [COUNTER, DURATION].freeze + + def self.detail_store + ::Gitlab::SafeRequestStore[DETAIL_STORE] ||= [] + end + + def self.duration + Gitlab::SafeRequestStore[DURATION].to_f + end + + def self.request_count + Gitlab::SafeRequestStore[COUNTER].to_i + end + + def self.payload + { + COUNTER => request_count, + DURATION => duration + } + end + + def request(event) + payload = event.payload + add_to_detail_store(payload) + add_to_request_store(payload) + expose_metrics(payload) + end + + private + + def current_transaction + ::Gitlab::Metrics::Transaction.current + end + + def add_to_detail_store(payload) + return unless Gitlab::PerformanceBar.enabled_for_request? + + self.class.detail_store << { + duration: payload[:duration], + scheme: payload[:scheme], + method: payload[:method], + host: payload[:host], + port: payload[:port], + path: payload[:path], + query: payload[:query], + code: payload[:code], + exception_object: payload[:exception_object], + backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller) + } + end + + def add_to_request_store(payload) + return unless Gitlab::SafeRequestStore.active? + + Gitlab::SafeRequestStore[COUNTER] = Gitlab::SafeRequestStore[COUNTER].to_i + 1 + Gitlab::SafeRequestStore[DURATION] = Gitlab::SafeRequestStore[DURATION].to_f + payload[:duration].to_f + end + + def expose_metrics(payload) + return unless current_transaction + + labels = { method: payload[:method], code: payload[:code] || DEFAULT_STATUS_CODE } + + current_transaction.increment(:gitlab_external_http_total, 1, labels) do + docstring 'External HTTP calls' + label_keys labels.keys + end + + current_transaction.observe(:gitlab_external_http_duration_seconds, payload[:duration]) do + docstring 'External HTTP time' + buckets [0.001, 0.01, 0.1, 1.0, 2.0, 5.0] + end + + if payload[:exception_object].present? + current_transaction.increment(:gitlab_external_http_exception_total, 1) do + docstring 'External HTTP exceptions' + end + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb new file mode 100644 index 00000000000..2791a39fb16 --- /dev/null +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module Subscribers + # - Adds logging for all Rack Attack blocks and throttling events. + # - Instrument the cache operations of RackAttack to use in structured + # logs. Two fields are exposed: + # + rack_attack_redis_count: the number of redis calls triggered by + # RackAttack in a request. + # + rack_attack_redis_duration_s: the total duration of all redis calls + # triggered by RackAttack in a request. + class RackAttack < ActiveSupport::Subscriber + attach_to 'rack_attack' + + INSTRUMENTATION_STORE_KEY = :rack_attack_instrumentation + + THROTTLES_WITH_USER_INFORMATION = [ + :throttle_authenticated_api, + :throttle_authenticated_web, + :throttle_authenticated_protected_paths_api, + :throttle_authenticated_protected_paths_web + ].freeze + + PAYLOAD_KEYS = [ + :rack_attack_redis_count, + :rack_attack_redis_duration_s + ].freeze + + def self.payload + Gitlab::SafeRequestStore[INSTRUMENTATION_STORE_KEY] ||= { + rack_attack_redis_count: 0, + rack_attack_redis_duration_s: 0.0 + } + end + + def redis(event) + self.class.payload[:rack_attack_redis_count] += 1 + self.class.payload[:rack_attack_redis_duration_s] += event.duration.to_f / 1000 + end + + def safelist(event) + req = event.payload[:request] + Gitlab::Instrumentation::Throttle.safelist = req.env['rack.attack.matched'] + end + + def throttle(event) + log_into_auth_logger(event) + end + + def blocklist(event) + log_into_auth_logger(event) + end + + def track(event) + log_into_auth_logger(event) + end + + private + + def log_into_auth_logger(event) + req = event.payload[:request] + rack_attack_info = { + message: 'Rack_Attack', + env: req.env['rack.attack.match_type'], + remote_ip: req.ip, + request_method: req.request_method, + path: req.fullpath, + matched: req.env['rack.attack.matched'] + } + + if THROTTLES_WITH_USER_INFORMATION.include? req.env['rack.attack.matched'].to_sym + user_id = req.env['rack.attack.match_discriminator'] + user = User.find_by(id: user_id) # rubocop:disable CodeReuse/ActiveRecord + + rack_attack_info[:user_id] = user_id + rack_attack_info['meta.user'] = user.username unless user.nil? + end + + Gitlab::InstrumentationHelper.add_instrumentation_data(rack_attack_info) + + logger.error(rack_attack_info) + end + + def logger + Gitlab::AuthLogger + end + end + end + end +end diff --git a/lib/gitlab/middleware/request_context.rb b/lib/gitlab/middleware/request_context.rb index 953423b371c..07f6f87a68c 100644 --- a/lib/gitlab/middleware/request_context.rb +++ b/lib/gitlab/middleware/request_context.rb @@ -16,9 +16,7 @@ module Gitlab # load balancer's IP. req = Rack::Request.new(env) - Gitlab::RequestContext.instance.client_ip = req.ip - Gitlab::RequestContext.instance.start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time - Gitlab::RequestContext.instance.request_start_time = Gitlab::Metrics::System.real_time + ::Gitlab::InstrumentationHelper.init_instrumentation_data(request_ip: req.ip) @app.call(env) end diff --git a/lib/gitlab/pages_transfer.rb b/lib/gitlab/pages_transfer.rb index 8ae0ec5a78a..c1ccfae3e1f 100644 --- a/lib/gitlab/pages_transfer.rb +++ b/lib/gitlab/pages_transfer.rb @@ -7,16 +7,26 @@ # module Gitlab class PagesTransfer < ProjectTransfer - class Async - METHODS = %w[move_namespace move_project rename_project rename_namespace].freeze + METHODS = %w[move_namespace move_project rename_project rename_namespace].freeze + class Async METHODS.each do |meth| define_method meth do |*args| + next unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) + PagesTransferWorker.perform_async(meth, args) end end end + METHODS.each do |meth| + define_method meth do |*args| + next unless Feature.enabled?(:pages_update_legacy_storage, default_enabled: true) + + super(*args) + end + end + def async @async ||= Async.new end diff --git a/lib/gitlab/patch/prependable.rb b/lib/gitlab/patch/prependable.rb index 22ece0a6a8b..dde78cd9178 100644 --- a/lib/gitlab/patch/prependable.rb +++ b/lib/gitlab/patch/prependable.rb @@ -39,9 +39,14 @@ module Gitlab def class_methods super + class_methods_module = const_get(:ClassMethods, false) + if instance_variable_defined?(:@_prepended_class_methods) - const_get(:ClassMethods, false).prepend @_prepended_class_methods + class_methods_module.prepend @_prepended_class_methods end + + # Hack to resolve https://gitlab.com/gitlab-org/gitlab/-/issues/23932 + extend class_methods_module if ENV['STATIC_VERIFICATION'] end def prepended(base = nil, &block) diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb index d1504d88315..380340b80be 100644 --- a/lib/gitlab/performance_bar/stats.rb +++ b/lib/gitlab/performance_bar/stats.rb @@ -27,27 +27,40 @@ module Gitlab end def log_sql_queries(id, data) - return [] unless queries = data.dig('data', 'active-record', 'details') - - queries.each do |query| - next unless location = parse_backtrace(query['backtrace']) + queries_by_location(data).each do |location, queries| + next unless location - log_info = location.merge( + duration = queries.sum { |query| query['duration'].to_f } + log_info = { + method_path: "#{location[:filename]}:#{location[:method]}", + filename: location[:filename], type: :sql, request_id: id, - duration_ms: query['duration'].to_f - ) + count: queries.count, + duration_ms: duration + } logger.info(log_info) end end + def queries_by_location(data) + return [] unless queries = data.dig('data', 'active-record', 'details') + + queries.group_by do |query| + parse_backtrace(query['backtrace']) + end + end + def parse_backtrace(backtrace) return unless match = /(?<filename>.*):(?<filenum>\d+):in `(?<method>.*)'/.match(backtrace.first) { filename: match[:filename], - filenum: match[:filenum].to_i, + # filenum may change quite frequently with every change in the file, + # because the intention is to aggregate these queries, we group + # them rather by method name which should not change so frequently + # filenum: match[:filenum].to_i, method: match[:method] } end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index b56fd8278a1..6a404c34044 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -181,13 +181,12 @@ module Gitlab end types MergeRequest condition do - Feature.enabled?(:merge_request_reviewers, project, default_enabled: :yaml) && - current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |reviewer_param| extract_users(reviewer_param) end - command :assign_reviewer, :reviewer do |users| + command :assign_reviewer, :reviewer, :request_review do |users| next if users.empty? if quick_action_target.allows_multiple_reviewers? @@ -221,7 +220,6 @@ module Gitlab types MergeRequest condition do quick_action_target.persisted? && - Feature.enabled?(:merge_request_reviewers, project, default_enabled: :yaml) && quick_action_target.reviewers.any? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index 2a94fb91880..ae3c89c3565 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -12,13 +12,15 @@ module Gitlab rack_attack::Request.include(Gitlab::RackAttack::Request) # This is Rack::Attack::DEFAULT_THROTTLED_RESPONSE, modified to allow a custom response - Rack::Attack.throttled_response = lambda do |env| + rack_attack.throttled_response = lambda do |env| throttled_headers = Gitlab::RackAttack.throttled_response_headers( env['rack.attack.matched'], env['rack.attack.match_data'] ) [429, { 'Content-Type' => 'text/plain' }.merge(throttled_headers), [Gitlab::Throttle.rate_limiting_response_text]] end + rack_attack.cache.store = Gitlab::RackAttack::InstrumentedCacheStore.new + # Configure the throttles configure_throttles(rack_attack) diff --git a/lib/gitlab/rack_attack/instrumented_cache_store.rb b/lib/gitlab/rack_attack/instrumented_cache_store.rb new file mode 100644 index 00000000000..8cf9082384f --- /dev/null +++ b/lib/gitlab/rack_attack/instrumented_cache_store.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module RackAttack + # This class is a proxy for all Redis calls made by RackAttack. All the + # calls are instrumented, then redirected to ::Rails.cache. This class + # instruments the standard interfaces of ActiveRecord::Cache defined in + # https://github.com/rails/rails/blob/v6.0.3.1/activesupport/lib/active_support/cache.rb#L315 + # + # For more information, please see + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/751 + class InstrumentedCacheStore + NOTIFICATION_CHANNEL = 'redis.rack_attack' + + delegate :silence!, :mute, to: :@upstream_store + + def initialize(upstream_store: ::Rails.cache, notifier: ActiveSupport::Notifications) + @upstream_store = upstream_store + @notifier = notifier + end + + [:fetch, :read, :read_multi, :write_multi, :fetch_multi, :write, :delete, + :exist?, :delete_matched, :increment, :decrement, :cleanup, :clear].each do |interface| + define_method interface do |*args, **k_args, &block| + @notifier.instrument(NOTIFICATION_CHANNEL, operation: interface) do + @upstream_store.public_send(interface, *args, **k_args, &block) # rubocop:disable GitlabSecurity/PublicSend + end + end + end + end + end +end diff --git a/lib/gitlab/recaptcha.rb b/lib/gitlab/recaptcha.rb index f3cbe1db901..a08cea5a435 100644 --- a/lib/gitlab/recaptcha.rb +++ b/lib/gitlab/recaptcha.rb @@ -2,8 +2,10 @@ module Gitlab module Recaptcha + extend Gitlab::Utils::StrongMemoize + def self.load_configurations! - if Gitlab::CurrentSettings.recaptcha_enabled || enabled_on_login? + if enabled? || enabled_on_login? ::Recaptcha.configure do |config| config.site_key = Gitlab::CurrentSettings.recaptcha_site_key config.secret_key = Gitlab::CurrentSettings.recaptcha_private_key diff --git a/lib/gitlab/relative_positioning.rb b/lib/gitlab/relative_positioning.rb index b5a923f0824..e2cbe4b2de0 100644 --- a/lib/gitlab/relative_positioning.rb +++ b/lib/gitlab/relative_positioning.rb @@ -13,5 +13,18 @@ module Gitlab MIN_GAP = 2 NoSpaceLeft = Class.new(StandardError) + IllegalRange = Class.new(ArgumentError) + + def self.range(lhs, rhs) + if lhs && rhs + ClosedRange.new(lhs, rhs) + elsif lhs + StartingFrom.new(lhs) + elsif rhs + EndingAt.new(rhs) + else + raise IllegalRange, 'One of rhs or lhs must be provided' unless lhs && rhs + end + end end end diff --git a/lib/gitlab/relative_positioning/range.rb b/lib/gitlab/relative_positioning/range.rb index 174d5ef4b35..0b0ccdf5be4 100644 --- a/lib/gitlab/relative_positioning/range.rb +++ b/lib/gitlab/relative_positioning/range.rb @@ -2,8 +2,6 @@ module Gitlab module RelativePositioning - IllegalRange = Class.new(ArgumentError) - class Range attr_reader :lhs, :rhs @@ -34,18 +32,6 @@ module Gitlab end end - def self.range(lhs, rhs) - if lhs && rhs - ClosedRange.new(lhs, rhs) - elsif lhs - StartingFrom.new(lhs) - elsif rhs - EndingAt.new(rhs) - else - raise IllegalRange, 'One of rhs or lhs must be provided' unless lhs && rhs - end - end - class ClosedRange < RelativePositioning::Range def initialize(lhs, rhs) @lhs, @rhs = lhs, rhs diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb index 952ae55d90a..c9eefe9a647 100644 --- a/lib/gitlab/request_context.rb +++ b/lib/gitlab/request_context.rb @@ -7,7 +7,7 @@ module Gitlab RequestDeadlineExceeded = Class.new(StandardError) - attr_accessor :client_ip, :start_thread_cpu_time, :request_start_time + attr_accessor :client_ip, :start_thread_cpu_time, :request_start_time, :thread_memory_allocations class << self def instance diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index 79562a8223b..a84a6ac2d14 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -23,7 +23,9 @@ module Gitlab end def self.verified?(env) - call(env) + minimal_env = env.slice('REQUEST_METHOD', 'rack.session', 'HTTP_X_CSRF_TOKEN') + .merge('rack.input' => '') + call(minimal_env) true rescue ActionController::InvalidAuthenticityToken diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 8b40aaa101a..647ac169f05 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -81,6 +81,10 @@ module Gitlab puma? || sidekiq? || action_cable? end + def puma_in_clustered_mode? + puma? && Puma.cli_config.options[:workers].to_i > 0 + end + def max_threads threads = 1 # main thread diff --git a/lib/gitlab/sample_data_template.rb b/lib/gitlab/sample_data_template.rb index 06ea53e4018..5a2a7d46bdf 100644 --- a/lib/gitlab/sample_data_template.rb +++ b/lib/gitlab/sample_data_template.rb @@ -5,7 +5,7 @@ module Gitlab class << self def localized_templates_table [ - SampleDataTemplate.new('sample', 'Sample GitLab Project', _('Get started with a project that follows best practices for setting up GitLab for your own organization, including sample Issues, Merge Requests, and Milestones'), 'https://gitlab.com/gitlab-org/sample-data-templates/sample-gitlab-project') + SampleDataTemplate.new('sample', 'Sample GitLab Project', _('An example project that shows off the best practices for setting up GitLab for your own organization, including sample issues, merge requests, and milestones'), 'https://gitlab.com/gitlab-org/sample-data-templates/sample-gitlab-project') ].freeze end diff --git a/lib/gitlab/search/query.rb b/lib/gitlab/search/query.rb index 5b1f9400bc7..c0420126ada 100644 --- a/lib/gitlab/search/query.rb +++ b/lib/gitlab/search/query.rb @@ -5,6 +5,9 @@ module Gitlab class Query < SimpleDelegator include EncodingHelper + QUOTES_REGEXP = %r{\A"|"\Z}.freeze + TOKEN_WITH_QUOTES_REGEXP = %r{\s(?=(?:[^"]|"[^"]*")*$)}.freeze + def initialize(query, filter_opts = {}, &block) @raw_query = query.dup @filters = [] @@ -35,22 +38,24 @@ module Gitlab def extract_filters fragments = [] + query_tokens = parse_raw_query filters = @filters.each_with_object([]) do |filter, parsed_filters| - match = @raw_query.split.find { |part| part =~ /\A-?#{filter[:name]}:/ } + match = query_tokens.find { |part| part =~ /\A-?#{filter[:name]}:/ } + next unless match input = match.split(':')[1..-1].join next if input.empty? filter[:negated] = match.start_with?("-") - filter[:value] = parse_filter(filter, input) + filter[:value] = parse_filter(filter, input.gsub(QUOTES_REGEXP, '')) filter[:regex_value] = Regexp.escape(filter[:value]).gsub('\*', '.*?') fragments << match parsed_filters << filter end - query = (@raw_query.split - fragments).join(' ') + query = (query_tokens - fragments).join(' ') query = '*' if query.empty? [query, filters] @@ -61,6 +66,13 @@ module Gitlab @filter_options[:encode_binary] ? encode_binary(result) : result end + + def parse_raw_query + # Positive lookahead for any non-quote char or even number of quotes + # for example '"search term" path:"foo bar.txt"' would break into + # ["search term", "path:\"foo bar.txt\""] + @raw_query.split(TOKEN_WITH_QUOTES_REGEXP).reject(&:empty?) + end end end end diff --git a/lib/gitlab/search/sort_options.rb b/lib/gitlab/search/sort_options.rb index 3395c34d171..2ab38147462 100644 --- a/lib/gitlab/search/sort_options.rb +++ b/lib/gitlab/search/sort_options.rb @@ -11,6 +11,10 @@ module Gitlab :created_at_asc when %w[created_at desc], [nil, 'created_desc'] :created_at_desc + when %w[updated_at asc], [nil, 'updated_asc'] + :updated_at_asc + when %w[updated_at desc], [nil, 'updated_desc'] + :updated_at_desc else :unknown end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 0091ae1e8ce..d0beb74c289 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -136,6 +136,10 @@ module Gitlab scope.reorder('created_at ASC') when :created_at_desc scope.reorder('created_at DESC') + when :updated_at_asc + scope.reorder('updated_at ASC') + when :updated_at_desc + scope.reorder('updated_at DESC') else scope.reorder('created_at DESC') end diff --git a/lib/gitlab/sidekiq_death_handler.rb b/lib/gitlab/sidekiq_death_handler.rb index f86d9f17b5f..91bfc7dca80 100644 --- a/lib/gitlab/sidekiq_death_handler.rb +++ b/lib/gitlab/sidekiq_death_handler.rb @@ -6,7 +6,7 @@ module Gitlab include ::Gitlab::SidekiqMiddleware::MetricsHelper def handler(job, _exception) - labels = create_labels(job['class'].constantize, job['queue']) + labels = create_labels(job['class'].constantize, job['queue'], job) counter.increment(labels) end diff --git a/lib/gitlab/sidekiq_logging/exception_handler.rb b/lib/gitlab/sidekiq_logging/exception_handler.rb deleted file mode 100644 index 8ae6addc2c6..00000000000 --- a/lib/gitlab/sidekiq_logging/exception_handler.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SidekiqLogging - class ExceptionHandler - def call(job_exception, context) - data = { - error_class: job_exception.class.name, - error_message: job_exception.message - } - - if context.is_a?(Hash) - data.merge!(context) - # correlation_id, jid, and class are available inside the job - # Hash, so promote these arguments to the root tree so that - # can be searched alongside other Sidekiq log messages. - job_data = data.delete(:job) - data.merge!(job_data) if job_data.present? - end - - data[:error_backtrace] = Rails.backtrace_cleaner.clean(job_exception.backtrace) if job_exception.backtrace.present? - - Sidekiq.logger.warn(data) - end - end - end -end diff --git a/lib/gitlab/sidekiq_logging/logs_jobs.rb b/lib/gitlab/sidekiq_logging/logs_jobs.rb index dc81c34c4d0..6f8cc1c60e9 100644 --- a/lib/gitlab/sidekiq_logging/logs_jobs.rb +++ b/lib/gitlab/sidekiq_logging/logs_jobs.rb @@ -12,6 +12,7 @@ module Gitlab # Error information from the previous try is in the payload for # displaying in the Sidekiq UI, but is very confusing in logs! job = job.except('error_backtrace', 'error_class', 'error_message') + job['class'] = job.delete('wrapped') if job['wrapped'].present? # Add process id params job['pid'] = ::Process.pid diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index eb845c5ff8d..654b17c5740 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -11,13 +11,25 @@ module Gitlab def call(job, queue) started_time = get_time base_payload = parse_job(job) + ActiveRecord::LogSubscriber.reset_runtime - Sidekiq.logger.info log_job_start(base_payload) + Sidekiq.logger.info log_job_start(job, base_payload) yield Sidekiq.logger.info log_job_done(job, started_time, base_payload) + rescue Sidekiq::JobRetry::Handled => job_exception + # Sidekiq::JobRetry::Handled is raised by the internal Sidekiq + # processor. It is a wrapper around real exception indicating an + # exception is already handled by the Job retrier. The real exception + # should be unwrapped before being logged. + # + # For more information: + # https://github.com/mperham/sidekiq/blob/v5.2.7/lib/sidekiq/processor.rb#L173 + Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception.cause || job_exception) + + raise rescue => job_exception Sidekiq.logger.warn log_job_done(job, started_time, base_payload, job_exception) @@ -27,7 +39,9 @@ module Gitlab private def add_instrumentation_keys!(job, output_payload) - output_payload.merge!(job.slice(*::Gitlab::InstrumentationHelper.keys)) + instrumentation_values = job.slice(*::Gitlab::InstrumentationHelper.keys).stringify_keys + + output_payload.merge!(instrumentation_values) end def add_logging_extras!(job, output_payload) @@ -36,17 +50,15 @@ module Gitlab ) end - def add_db_counters!(job, output_payload) - output_payload.merge!(job.slice(*::Gitlab::Metrics::Subscribers::ActiveRecord::DB_COUNTERS)) - end - - def log_job_start(payload) + def log_job_start(job, payload) payload['message'] = "#{base_message(payload)}: start" payload['job_status'] = 'start' scheduling_latency_s = ::Gitlab::InstrumentationHelper.queue_duration_for_job(payload) payload['scheduling_latency_s'] = scheduling_latency_s if scheduling_latency_s + payload['job_size_bytes'] = Sidekiq.dump_json(job).bytesize + payload end @@ -54,7 +66,6 @@ module Gitlab payload = payload.dup add_instrumentation_keys!(job, payload) add_logging_extras!(job, payload) - add_db_counters!(job, payload) elapsed_time = elapsed(started_time) add_time_keys!(elapsed_time, payload) @@ -66,6 +77,7 @@ module Gitlab payload['job_status'] = 'fail' payload['error_message'] = job_exception.message payload['error_class'] = job_exception.class.name + add_exception_backtrace!(job_exception, payload) else payload['message'] = "#{message}: done: #{payload['duration_s']} sec" payload['job_status'] = 'done' @@ -79,26 +91,22 @@ module Gitlab def add_time_keys!(time, payload) payload['duration_s'] = time[:duration].round(Gitlab::InstrumentationHelper::DURATION_PRECISION) - - # ignore `cpu_s` if the platform does not support Process::CLOCK_THREAD_CPUTIME_ID (time[:cputime] == 0) - # supported OS version can be found at: https://www.rubydoc.info/stdlib/core/2.1.6/Process:clock_gettime - payload['cpu_s'] = time[:cputime].round(Gitlab::InstrumentationHelper::DURATION_PRECISION) if time[:cputime] > 0 payload['completed_at'] = Time.now.utc.to_f end + def add_exception_backtrace!(job_exception, payload) + return if job_exception.backtrace.blank? + + payload['error_backtrace'] = Rails.backtrace_cleaner.clean(job_exception.backtrace) + end + def elapsed(t0) t1 = get_time - { - duration: t1[:now] - t0[:now], - cputime: t1[:thread_cputime] - t0[:thread_cputime] - } + { duration: t1[:now] - t0[:now] } end def get_time - { - now: current_time, - thread_cputime: defined?(Process::CLOCK_THREAD_CPUTIME_ID) ? Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID) : 0 - } + { now: current_time } end def current_time diff --git a/lib/gitlab/sidekiq_middleware/client_metrics.rb b/lib/gitlab/sidekiq_middleware/client_metrics.rb index 7ee8a623d30..6bc08a97c07 100644 --- a/lib/gitlab/sidekiq_middleware/client_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/client_metrics.rb @@ -11,10 +11,10 @@ module Gitlab @metrics = init_metrics end - def call(worker_class, _job, queue, _redis_pool) + def call(worker_class, job, queue, _redis_pool) # worker_class can either be the string or class of the worker being enqueued. worker_class = worker_class.safe_constantize if worker_class.respond_to?(:safe_constantize) - labels = create_labels(worker_class, queue) + labels = create_labels(worker_class, queue, job) @metrics.fetch(ENQUEUED).increment(labels, 1) diff --git a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb index 979a3fce7e6..a66a4de4655 100644 --- a/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb +++ b/lib/gitlab/sidekiq_middleware/instrumentation_logger.rb @@ -4,8 +4,11 @@ module Gitlab module SidekiqMiddleware class InstrumentationLogger def call(worker, job, queue) + ::Gitlab::InstrumentationHelper.init_instrumentation_data + yield + ensure # The Sidekiq logger is called outside the middleware block, so # we need to modify the job hash to pass along this information # since RequestStore is only active in the Sidekiq middleware. diff --git a/lib/gitlab/sidekiq_middleware/metrics_helper.rb b/lib/gitlab/sidekiq_middleware/metrics_helper.rb index 5c1ce2b98e8..60e79ee1188 100644 --- a/lib/gitlab/sidekiq_middleware/metrics_helper.rb +++ b/lib/gitlab/sidekiq_middleware/metrics_helper.rb @@ -8,9 +8,11 @@ module Gitlab private - def create_labels(worker_class, queue) + def create_labels(worker_class, queue, job) + worker_name = (job['wrapped'].presence || worker_class).to_s + labels = { queue: queue.to_s, - worker: worker_class.to_s, + worker: worker_name, urgency: "", external_dependencies: FALSE_LABEL, feature_category: "", diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 7f3048f4c6e..4ab8d313ad8 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -20,7 +20,7 @@ module Gitlab # in metrics and can use them in the `ThreadsSampler` for setting a label Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME - labels = create_labels(worker.class, queue) + labels = create_labels(worker.class, queue, job) queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) @metrics[:sidekiq_jobs_queue_duration_seconds].observe(labels, queue_duration) if queue_duration diff --git a/lib/gitlab/suggestions/commit_message.rb b/lib/gitlab/suggestions/commit_message.rb index d59a8fc3730..5bca3efe6e1 100644 --- a/lib/gitlab/suggestions/commit_message.rb +++ b/lib/gitlab/suggestions/commit_message.rb @@ -6,14 +6,15 @@ module Gitlab DEFAULT_SUGGESTION_COMMIT_MESSAGE = 'Apply %{suggestions_count} suggestion(s) to %{files_count} file(s)' - def initialize(user, suggestion_set) + def initialize(user, suggestion_set, custom_message = nil) @user = user @suggestion_set = suggestion_set + @custom_message = custom_message end def message project = suggestion_set.project - user_defined_message = project.suggestion_commit_message.presence + user_defined_message = @custom_message.presence || project.suggestion_commit_message.presence message = user_defined_message || DEFAULT_SUGGESTION_COMMIT_MESSAGE Gitlab::StringPlaceholderReplacer diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index c702c6f1add..db3c058184c 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -66,6 +66,18 @@ module Gitlab answer end + # Prompt the user to input a password + # + # message - custom message to display before input + def prompt_for_password(message = 'Enter password: ') + unless STDIN.tty? + print(message) + return STDIN.gets.chomp + end + + STDIN.getpass(message) + end + # Runs the given command and matches the output against the given pattern # # Returns nil if nothing matched diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index b659bff52ad..0f933a61598 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -8,6 +8,7 @@ module Gitlab def initialize(path, project = nil, category: nil) @path = path @category = category + @project = project @finder = self.class.finder(project) end @@ -31,6 +32,10 @@ module Gitlab # override with a comment to be placed at the top of the blob. end + def project_id + @project&.id + end + # Present for compatibility with license templates, which can replace text # like `[fullname]` with a user-specified string. This is a no-op for # other templates @@ -76,7 +81,7 @@ module Gitlab end # Defines which strategy will be used to get templates files - # RepoTemplateFinder - Finds templates on project repository, templates are filtered perproject + # RepoTemplateFinder - Finds templates on project repository, templates are filtered per project # GlobalTemplateFinder - Finds templates on gitlab installation source, templates can be used in all projects def finder(project = nil) raise NotImplementedError @@ -95,19 +100,29 @@ module Gitlab File.join(base_dir, categories[category]) end - # If template is organized by category it returns { category_name: [{ name: template_name }, { name: template2_name }] } - # If no category is present returns [{ name: template_name }, { name: template2_name}] - def dropdown_names(project = nil) - return [] if project && !project.repository.exists? + # `repository_template_names` - reads through Gitaly the actual templates names within a + # given project's repository. This is only used by issue and merge request templates, + # that need to call this once and then cache the returned value. + # + # `template_names` - is an alias to `repository_template_names`. It would read through + # Gitaly the actual template names within a given project's repository for all file templates + # other than `issue` and `merge request` description templates, which would instead + # overwrite the `template_names` method to return a redis cached version, by reading cached values + # from `repository.issue_template_names_by_category` and `repository.merge_request_template_names_by_category` + # methods. + def repository_template_names(project) + template_names_by_category(self.all(project)) + end + alias_method :template_names, :repository_template_names - if categories.any? - categories.keys.map do |category| - files = self.by_category(category, project) - [category, files.map { |t| { name: t.name } }] - end.to_h - else - files = self.all(project) - files.map { |t| { name: t.name } } + def template_names_by_category(items) + grouped = items.group_by(&:category) + categories = grouped.keys + + categories.each_with_object({}) do |category, hash| + hash[category] = grouped[category].map do |item| + { name: item.name, id: item.key, key: item.key, project_id: item.try(:project_id) } + end end end diff --git a/lib/gitlab/template/finders/global_template_finder.rb b/lib/gitlab/template/finders/global_template_finder.rb index 9b39d386674..6d2677175e6 100644 --- a/lib/gitlab/template/finders/global_template_finder.rb +++ b/lib/gitlab/template/finders/global_template_finder.rb @@ -5,9 +5,10 @@ module Gitlab module Template module Finders class GlobalTemplateFinder < BaseTemplateFinder - def initialize(base_dir, extension, categories = {}, excluded_patterns: []) + def initialize(base_dir, extension, categories = {}, include_categories_for_file = {}, excluded_patterns: []) @categories = categories @extension = extension + @include_categories_for_file = include_categories_for_file @excluded_patterns = excluded_patterns super(base_dir) @@ -47,7 +48,9 @@ module Gitlab end def select_directory(file_name) - @categories.keys.find do |category| + categories = @categories + categories.merge!(@include_categories_for_file[file_name]) if @include_categories_for_file[file_name].present? + categories.keys.find do |category| File.exist?(File.join(category_directory(category), file_name)) end end diff --git a/lib/gitlab/template/finders/repo_template_finder.rb b/lib/gitlab/template/finders/repo_template_finder.rb index 8e234148a63..9f0ba97bcdf 100644 --- a/lib/gitlab/template/finders/repo_template_finder.rb +++ b/lib/gitlab/template/finders/repo_template_finder.rb @@ -11,8 +11,8 @@ module Gitlab def initialize(project, base_dir, extension, categories = {}) @categories = categories @extension = extension - @repository = project.repository - @commit = @repository.head_commit if @repository.exists? + @repository = project&.repository + @commit = @repository.head_commit if @repository&.exists? super(base_dir) end @@ -51,7 +51,7 @@ module Gitlab private def select_directory(file_name) - return [] unless @commit + return unless @commit # Insert root as directory directories = ["", *@categories.keys] diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index c295cc75da5..01158cafc4f 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -25,6 +25,12 @@ module Gitlab } end + def include_categories_for_file + { + "SAST#{self.extension}" => { 'Security' => 'Security' } + } + end + def excluded_patterns strong_memoize(:excluded_patterns) do BASE_EXCLUDED_PATTERNS + additional_excluded_patterns @@ -41,7 +47,11 @@ module Gitlab def finder(project = nil) Gitlab::Template::Finders::GlobalTemplateFinder.new( - self.base_dir, self.extension, self.categories, excluded_patterns: self.excluded_patterns + self.base_dir, + self.extension, + self.categories, + self.include_categories_for_file, + excluded_patterns: self.excluded_patterns ) end end diff --git a/lib/gitlab/template/issue_template.rb b/lib/gitlab/template/issue_template.rb index 01b191733d4..3049f43b322 100644 --- a/lib/gitlab/template/issue_template.rb +++ b/lib/gitlab/template/issue_template.rb @@ -15,6 +15,16 @@ module Gitlab def finder(project) Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories) end + + def template_names(project) + return {} unless project&.repository&.exists? + + # here we rely on project.repository caching mechanism. Ideally we would want the template finder to have its + # own caching mechanism to avoid the back and forth call jumps between finder and model. + # + # follow-up issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300279 + project.repository.issue_template_names_by_category + end end end end diff --git a/lib/gitlab/template/merge_request_template.rb b/lib/gitlab/template/merge_request_template.rb index 357b31cd82e..9442f3b13fb 100644 --- a/lib/gitlab/template/merge_request_template.rb +++ b/lib/gitlab/template/merge_request_template.rb @@ -15,6 +15,16 @@ module Gitlab def finder(project) Gitlab::Template::Finders::RepoTemplateFinder.new(project, self.base_dir, self.extension, self.categories) end + + def template_names(project) + return {} unless project&.repository&.exists? + + # here we rely on project.repository caching mechanism. Ideally we would want the template finder to have its + # own caching mechanism to avoid the back and forth call jumps between finder and model. + # + # follow-up issue: https://gitlab.com/gitlab-org/gitlab/-/issues/300279 + project.repository.merge_request_template_names_by_category + end end end end diff --git a/lib/gitlab/terraform/state_migration_helper.rb b/lib/gitlab/terraform/state_migration_helper.rb new file mode 100644 index 00000000000..04c1cbd0373 --- /dev/null +++ b/lib/gitlab/terraform/state_migration_helper.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Terraform + class StateMigrationHelper + class << self + def migrate_to_remote_storage(&block) + migrate_in_batches( + ::Terraform::StateVersion.with_files_stored_locally.preload_state, + ::Terraform::StateUploader::Store::REMOTE, + &block + ) + end + + private + + def batch_size + ENV.fetch('MIGRATION_BATCH_SIZE', 10).to_i + end + + def migrate_in_batches(versions, store, &block) + versions.find_each(batch_size: batch_size) do |version| # rubocop:disable CodeReuse/ActiveRecord + version.file.migrate!(store) + + yield version if block_given? + end + end + end + end + end +end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index ca4afb4c19c..09697705361 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -24,8 +24,8 @@ module Gitlab Gitlab::CurrentSettings.snowplow_enabled? end - def event(category, action, label: nil, property: nil, value: nil, context: [], standard_context: nil) - context.push(standard_context.to_context) if standard_context + def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil) # rubocop:disable Metrics/ParameterLists + context += [Tracking::StandardContext.new(project: project, user: user, namespace: namespace).to_context] snowplow.event(category, action, label: label, property: property, value: value, context: context) product_analytics.event(category, action, label: label, property: property, value: value, context: context) diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 71dfe27dd5a..92fdd008249 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,38 +3,36 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-1'.freeze + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-3'.freeze + GITLAB_RAILS_SOURCE = 'gitlab-rails'.freeze - def initialize(namespace: nil, project: nil, **data) - @namespace = namespace - @project = project + def initialize(namespace: nil, project: nil, user: nil, **data) @data = data end - def namespace_id - namespace&.id + def to_context + SnowplowTracker::SelfDescribingJson.new(GITLAB_STANDARD_SCHEMA_URL, to_h) end - def project_id - @project&.id + def environment + return 'production' if Gitlab.com_and_canary? + + return 'staging' if Gitlab.staging? + + 'development' end - def to_context - SnowplowTracker::SelfDescribingJson.new(GITLAB_STANDARD_SCHEMA_URL, to_h) + def source + GITLAB_RAILS_SOURCE end private - def namespace - @namespace || @project&.namespace - end - def to_h - public_methods(false).each_with_object({}) do |method, hash| - next if method == :to_context - - hash[method] = public_send(method) # rubocop:disable GitlabSecurity/PublicSend - end.merge(@data) + { + environment: environment, + source: source + }.merge(@data) end end end diff --git a/lib/gitlab/usage/docs/helper.rb b/lib/gitlab/usage/docs/helper.rb new file mode 100644 index 00000000000..8483334800b --- /dev/null +++ b/lib/gitlab/usage/docs/helper.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Docs + # Helper with functions to be used by HAML templates + module Helper + HEADER = %w(field value).freeze + SKIP_KEYS = %i(description).freeze + + def auto_generated_comment + <<-MARKDOWN.strip_heredoc + --- + stage: Growth + group: Product Intelligence + info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers + --- + + <!--- + This documentation is auto generated by a script. + + Please do not edit this file directly, check generate_metrics_dictionary task on lib/tasks/gitlab/usage_data.rake. + ---> + + <!-- vale gitlab.Spelling = NO --> + MARKDOWN + end + + def render_name(name) + "## `#{name}`\n" + end + + def render_description(object) + object.description + end + + def render_attribute_row(key, value) + value = Gitlab::Usage::Docs::ValueFormatter.format(key, value) + table_row(["`#{key}`", value]) + end + + def render_attributes_table(object) + <<~MARKDOWN + + #{table_row(HEADER)} + #{table_row(HEADER.map { '---' })} + #{table_value_rows(object.attributes)} + MARKDOWN + end + + def table_value_rows(attributes) + attributes.reject { |k, _| k.in?(SKIP_KEYS) }.map do |key, value| + render_attribute_row(key, value) + end.join("\n") + end + + def table_row(array) + "| #{array.join(' | ')} |" + end + end + end + end +end diff --git a/lib/gitlab/usage/docs/renderer.rb b/lib/gitlab/usage/docs/renderer.rb new file mode 100644 index 00000000000..7a7c58005bb --- /dev/null +++ b/lib/gitlab/usage/docs/renderer.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Docs + class Renderer + include Gitlab::Usage::Docs::Helper + DICTIONARY_PATH = Rails.root.join('doc', 'development', 'usage_ping') + TEMPLATE_PATH = Rails.root.join('lib', 'gitlab', 'usage', 'docs', 'templates', 'default.md.haml') + + def initialize(metrics_definitions) + @layout = Haml::Engine.new(File.read(TEMPLATE_PATH)) + @metrics_definitions = metrics_definitions.sort + end + + def contents + # Render and remove an extra trailing new line + @contents ||= @layout.render(self, metrics_definitions: @metrics_definitions).sub!(/\n(?=\Z)/, '') + end + + def write + filename = DICTIONARY_PATH.join('dictionary.md').to_s + + FileUtils.mkdir_p(DICTIONARY_PATH) + File.write(filename, contents) + + filename + end + end + end + end +end diff --git a/lib/gitlab/usage/docs/templates/default.md.haml b/lib/gitlab/usage/docs/templates/default.md.haml new file mode 100644 index 00000000000..86e93be66c7 --- /dev/null +++ b/lib/gitlab/usage/docs/templates/default.md.haml @@ -0,0 +1,28 @@ += auto_generated_comment + +:plain + # Metrics Dictionary + + This file is autogenerated, please do not edit directly. + + To generate these files from the GitLab repository, run: + + ```shell + bundle exec rake gitlab:usage_data:generate_metrics_dictionary + ``` + + The Metrics Dictionary is based on the following metrics definition YAML files: + + - [`config/metrics`]('https://gitlab.com/gitlab-org/gitlab/-/tree/master/config/metrics') + - [`ee/config/metrics`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/config/metrics) + +Each table includes a `milestone`, which corresponds to the GitLab version when the metric +was released. +\ +- metrics_definitions.each do |name, object| + + = render_name(name) + + = render_description(object) + + = render_attributes_table(object) diff --git a/lib/gitlab/usage/docs/value_formatter.rb b/lib/gitlab/usage/docs/value_formatter.rb new file mode 100644 index 00000000000..a2dc9b081f8 --- /dev/null +++ b/lib/gitlab/usage/docs/value_formatter.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Docs + class ValueFormatter + def self.format(key, value) + case key + when :key_path + "**`#{value}`**" + when :data_source + value.to_s.capitalize + when :product_group + "`#{value}`" + when :introduced_by_url + "[Introduced by](#{value})" + when :distribution, :tier + Array(value).join(', ') + else + value + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb index e1648c78168..f3469209f48 100644 --- a/lib/gitlab/usage/metric.rb +++ b/lib/gitlab/usage/metric.rb @@ -7,16 +7,16 @@ module Gitlab InvalidMetricError = Class.new(RuntimeError) - attr_accessor :default_generation_path, :value + attr_accessor :key_path, :value - validates :default_generation_path, presence: true + validates :key_path, presence: true def definition - self.class.definitions[default_generation_path] + self.class.definitions[key_path] end - def unflatten_default_path - unflatten(default_generation_path.split('.'), value) + def unflatten_key_path + unflatten(key_path.split('.'), value) end class << self diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 96e572bb3db..01d202e4d45 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -13,9 +13,8 @@ module Gitlab @attributes = opts end - # The key is defined by default_generation and full_path def key - full_path[default_generation.to_sym] + key_path end def to_h @@ -23,8 +22,10 @@ module Gitlab end def validate! - self.class.schemer.validate(attributes.stringify_keys).map do |error| - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`")) + unless skip_validation? + self.class.schemer.validate(attributes.stringify_keys).each do |error| + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`")) + end end end @@ -79,6 +80,10 @@ module Gitlab def method_missing(method, *args) attributes[method] || super end + + def skip_validation? + !!attributes[:skip_validation] + end end end end diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb new file mode 100644 index 00000000000..1fc40798320 --- /dev/null +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Aggregates + UNION_OF_AGGREGATED_METRICS = 'OR' + INTERSECTION_OF_AGGREGATED_METRICS = 'AND' + ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze + AGGREGATED_METRICS_PATH = Rails.root.join('lib/gitlab/usage_data_counters/aggregated_metrics/*.yml') + AggregatedMetricError = Class.new(StandardError) + UnknownAggregationOperator = Class.new(AggregatedMetricError) + UnknownAggregationSource = Class.new(AggregatedMetricError) + + DATABASE_SOURCE = 'database' + REDIS_SOURCE = 'redis' + + SOURCES = { + DATABASE_SOURCE => Sources::PostgresHll, + REDIS_SOURCE => Sources::RedisHll + }.freeze + + class Aggregate + delegate :weekly_time_range, + :monthly_time_range, + to: Gitlab::UsageDataCounters::HLLRedisCounter + + def initialize(recorded_at) + @aggregated_metrics = load_metrics(AGGREGATED_METRICS_PATH) + @recorded_at = recorded_at + end + + def monthly_data + aggregated_metrics_data(**monthly_time_range) + end + + def weekly_data + aggregated_metrics_data(**weekly_time_range) + end + + private + + attr_accessor :aggregated_metrics, :recorded_at + + def aggregated_metrics_data(start_date:, end_date:) + aggregated_metrics.each_with_object({}) do |aggregation, data| + next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: :yaml, type: :development) + + case aggregation[:source] + when REDIS_SOURCE + data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) + when DATABASE_SOURCE + next unless Feature.enabled?('database_sourced_aggregated_metrics', default_enabled: false, type: :development) + + data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, start_date: start_date, end_date: end_date) + else + Gitlab::ErrorTracking + .track_and_raise_for_dev_exception(UnknownAggregationSource.new("Aggregation source: '#{aggregation[:source]}' must be included in #{SOURCES.keys}")) + + data[aggregation[:name]] = Gitlab::Utils::UsageData::FALLBACK + end + end + end + + def calculate_count_for_aggregation(aggregation:, start_date:, end_date:) + source = SOURCES[aggregation[:source]] + + case aggregation[:operator] + when UNION_OF_AGGREGATED_METRICS + source.calculate_metrics_union(metric_names: aggregation[:events], start_date: start_date, end_date: end_date, recorded_at: recorded_at) + when INTERSECTION_OF_AGGREGATED_METRICS + calculate_metrics_intersections(source: source, metric_names: aggregation[:events], start_date: start_date, end_date: end_date) + else + Gitlab::ErrorTracking + .track_and_raise_for_dev_exception(UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}")) + Gitlab::Utils::UsageData::FALLBACK + end + rescue Gitlab::UsageDataCounters::HLLRedisCounter::EventError, AggregatedMetricError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + Gitlab::Utils::UsageData::FALLBACK + end + + # calculate intersection of 'n' sets based on inclusion exclusion principle https://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle + # this method will be extracted to dedicated module with https://gitlab.com/gitlab-org/gitlab/-/issues/273391 + def calculate_metrics_intersections(source:, metric_names:, start_date:, end_date:, subset_powers_cache: Hash.new({})) + # calculate power of intersection of all given metrics from inclusion exclusion principle + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + + # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... + subset_powers_data = subsets_intersection_powers(source, metric_names, start_date, end_date, subset_powers_cache) + + # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| + power_of_union_of_all_metrics = begin + subset_powers_cache[metric_names.size][metric_names.join('_+_')] ||= \ + source.calculate_metrics_union(metric_names: metric_names, start_date: start_date, end_date: end_date, recorded_at: recorded_at) + end + + # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, + # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below + # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => + # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| + # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => + # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| + subset_powers_size_even = subset_powers_data.size.even? + + # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => + sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) + + # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| + sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_metrics : -power_of_union_of_all_metrics) + end + + def sum_subset_powers(subset_powers_data, subset_powers_size_even) + sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| + (index + 1).odd? ? value : -value + end + + (subset_powers_size_even ? -1 : 1) * sum_without_sign + end + + def subsets_intersection_powers(source, metric_names, start_date, end_date, subset_powers_cache) + subset_sizes = (1...metric_names.size) + + subset_sizes.map do |subset_size| + if subset_size > 1 + # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) + metric_names.combination(subset_size).sum do |metrics_subset| + subset_powers_cache[subset_size][metrics_subset.join('_&_')] ||= + calculate_metrics_intersections(source: source, metric_names: metrics_subset, start_date: start_date, end_date: end_date, subset_powers_cache: subset_powers_cache) + end + else + # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... + metric_names.sum do |metric| + subset_powers_cache[subset_size][metric] ||= \ + source.calculate_metrics_union(metric_names: metric, start_date: start_date, end_date: end_date, recorded_at: recorded_at) + end + end + end + end + + def load_metrics(wildcard) + Dir[wildcard].each_with_object([]) do |path, metrics| + metrics.push(*load_yaml_from_path(path)) + end + end + + def load_yaml_from_path(path) + YAML.safe_load(File.read(path), aliases: true)&.map(&:with_indifferent_access) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb new file mode 100644 index 00000000000..33678d2b813 --- /dev/null +++ b/lib/gitlab/usage/metrics/aggregates/sources/postgres_hll.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Aggregates + module Sources + class PostgresHll + class << self + def calculate_metrics_union(metric_names:, start_date:, end_date:, recorded_at:) + time_period = start_date && end_date ? (start_date..end_date) : nil + + Array(metric_names).each_with_object(Gitlab::Database::PostgresHll::Buckets.new) do |event, buckets| + json = read_aggregated_metric(metric_name: event, time_period: time_period, recorded_at: recorded_at) + raise UnionNotAvailable, "Union data not available for #{metric_names}" unless json + + buckets.merge_hash!(Gitlab::Json.parse(json)) + end.estimated_distinct_count + end + + def save_aggregated_metrics(metric_name:, time_period:, recorded_at_timestamp:, data:) + unless data.is_a? ::Gitlab::Database::PostgresHll::Buckets + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(StandardError.new("Unsupported data type: #{data.class}")) + return + end + + # Usage Ping report generation for gitlab.com is very long running process + # to make sure that saved keys are available at the end of report generation process + # lets use triple max generation time + keys_expiration = ::Gitlab::UsageData::MAX_GENERATION_TIME_FOR_SAAS * 3 + + Gitlab::Redis::SharedState.with do |redis| + redis.set( + redis_key(metric_name: metric_name, time_period: time_period&.values&.first, recorded_at: recorded_at_timestamp), + data.to_json, + ex: keys_expiration + ) + end + rescue ::Redis::CommandError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + + private + + def read_aggregated_metric(metric_name:, time_period:, recorded_at:) + Gitlab::Redis::SharedState.with do |redis| + redis.get(redis_key(metric_name: metric_name, time_period: time_period, recorded_at: recorded_at)) + end + end + + def redis_key(metric_name:, time_period:, recorded_at:) + # add timestamp at the end of the key to avoid stale keys if + # usage ping job is retried + "#{metric_name}_#{time_period_to_human_name(time_period)}-#{recorded_at.to_i}" + end + + def time_period_to_human_name(time_period) + return Gitlab::Utils::UsageData::ALL_TIME_PERIOD_HUMAN_NAME if time_period.blank? + + start_date = time_period.first.to_date + end_date = time_period.last.to_date + + if (end_date - start_date).to_i > 7 + Gitlab::Utils::UsageData::MONTHLY_PERIOD_HUMAN_NAME + else + Gitlab::Utils::UsageData::WEEKLY_PERIOD_HUMAN_NAME + end + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb new file mode 100644 index 00000000000..f3a4dcf1e31 --- /dev/null +++ b/lib/gitlab/usage/metrics/aggregates/sources/redis_hll.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Aggregates + module Sources + UnionNotAvailable = Class.new(AggregatedMetricError) + + class RedisHll + def self.calculate_metrics_union(metric_names:, start_date:, end_date:, recorded_at: nil) + union = Gitlab::UsageDataCounters::HLLRedisCounter + .calculate_events_union(event_names: metric_names, start_date: start_date, end_date: end_date) + + return union if union >= 0 + + raise UnionNotAvailable, "Union data not available for #{metric_names}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index f935c677930..8e096a9f351 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -12,6 +12,9 @@ # redis_usage_data { ::Gitlab::UsageCounters::PodLogs.usage_totals[:total] } module Gitlab class UsageData + DEPRECATED_VALUE = -1000 + MAX_GENERATION_TIME_FOR_SAAS = 40.hours + CE_MEMOIZED_VALUES = %i( issue_minimum_id issue_maximum_id @@ -23,6 +26,8 @@ module Gitlab deployment_minimum_id deployment_maximum_id auth_providers + aggregated_metrics + recorded_at ).freeze class << self @@ -75,7 +80,7 @@ module Gitlab end def recorded_at - Time.current + @recorded_at ||= Time.current end # rubocop: disable Metrics/AbcSize @@ -158,7 +163,7 @@ module Gitlab projects_with_repositories_enabled: count(ProjectFeature.where('repository_access_level > ?', ProjectFeature::DISABLED)), projects_with_tracing_enabled: count(ProjectTracingSetting), projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), - projects_with_alerts_service_enabled: count(AlertsService.active), + projects_with_alerts_service_enabled: count(Service.active.where(type: 'AlertsService')), projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), projects_with_prometheus_alerts: distinct_count(PrometheusAlert, :project_id), @@ -580,27 +585,35 @@ module Gitlab users_created: count(::User.where(time_period), start: user_minimum_id, finish: user_maximum_id), omniauth_providers: filtered_omniauth_provider_names.reject { |name| name == 'group_saml' }, user_auth_by_provider: distinct_count_user_auth_by_provider(time_period), + unique_users_all_imports: unique_users_all_imports(time_period), bulk_imports: { - gitlab: distinct_count(::BulkImport.where(time_period, source_type: :gitlab), :user_id) + gitlab: DEPRECATED_VALUE, + gitlab_v1: count(::BulkImport.where(time_period, source_type: :gitlab)) }, + project_imports: project_imports(time_period), + issue_imports: issue_imports(time_period), + group_imports: group_imports(time_period), + + # Deprecated data to be removed projects_imported: { - total: distinct_count(::Project.where(time_period).where.not(import_type: nil), :creator_id), - gitlab_project: projects_imported_count('gitlab_project', time_period), - gitlab: projects_imported_count('gitlab', time_period), - github: projects_imported_count('github', time_period), - bitbucket: projects_imported_count('bitbucket', time_period), - bitbucket_server: projects_imported_count('bitbucket_server', time_period), - gitea: projects_imported_count('gitea', time_period), - git: projects_imported_count('git', time_period), - manifest: projects_imported_count('manifest', time_period) + total: DEPRECATED_VALUE, + gitlab_project: DEPRECATED_VALUE, + gitlab: DEPRECATED_VALUE, + github: DEPRECATED_VALUE, + bitbucket: DEPRECATED_VALUE, + bitbucket_server: DEPRECATED_VALUE, + gitea: DEPRECATED_VALUE, + git: DEPRECATED_VALUE, + manifest: DEPRECATED_VALUE }, issues_imported: { - jira: distinct_count(::JiraImportState.where(time_period), :user_id), - fogbugz: projects_imported_count('fogbugz', time_period), - phabricator: projects_imported_count('phabricator', time_period), - csv: distinct_count(Issues::CsvImport.where(time_period), :user_id) + jira: DEPRECATED_VALUE, + fogbugz: DEPRECATED_VALUE, + phabricator: DEPRECATED_VALUE, + csv: DEPRECATED_VALUE }, - groups_imported: distinct_count(::GroupImportState.where(time_period), :user_id) + groups_imported: DEPRECATED_VALUE + # End of deprecated keys } end # rubocop: enable CodeReuse/ActiveRecord @@ -690,13 +703,13 @@ module Gitlab def aggregated_metrics_monthly { - aggregated_metrics: ::Gitlab::UsageDataCounters::HLLRedisCounter.aggregated_metrics_monthly_data + aggregated_metrics: aggregated_metrics.monthly_data } end def aggregated_metrics_weekly { - aggregated_metrics: ::Gitlab::UsageDataCounters::HLLRedisCounter.aggregated_metrics_weekly_data + aggregated_metrics: aggregated_metrics.weekly_data } end @@ -741,6 +754,10 @@ module Gitlab private + def aggregated_metrics + @aggregated_metrics ||= ::Gitlab::Usage::Metrics::Aggregates::Aggregate.new(recorded_at) + end + def event_monthly_active_users(date_range) data = { action_monthly_active_users_project_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::PUSH_ACTION, @@ -893,10 +910,52 @@ module Gitlab count relation, start: deployment_minimum_id, finish: deployment_maximum_id end + def project_imports(time_period) + { + gitlab_project: projects_imported_count('gitlab_project', time_period), + gitlab: projects_imported_count('gitlab', time_period), + github: projects_imported_count('github', time_period), + bitbucket: projects_imported_count('bitbucket', time_period), + bitbucket_server: projects_imported_count('bitbucket_server', time_period), + gitea: projects_imported_count('gitea', time_period), + git: projects_imported_count('git', time_period), + manifest: projects_imported_count('manifest', time_period), + gitlab_migration: count(::BulkImports::Entity.where(time_period).project_entity) # rubocop: disable CodeReuse/ActiveRecord + } + end + def projects_imported_count(from, time_period) - distinct_count(::Project.imported_from(from).where(time_period).where.not(import_type: nil), :creator_id) # rubocop: disable CodeReuse/ActiveRecord + count(::Project.imported_from(from).where(time_period).where.not(import_type: nil)) # rubocop: disable CodeReuse/ActiveRecord end + def issue_imports(time_period) + { + jira: count(::JiraImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord + fogbugz: projects_imported_count('fogbugz', time_period), + phabricator: projects_imported_count('phabricator', time_period), + csv: count(Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord + } + end + + def group_imports(time_period) + { + group_import: count(::GroupImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord + gitlab_migration: count(::BulkImports::Entity.where(time_period).group_entity) # rubocop: disable CodeReuse/ActiveRecord + } + end + + # rubocop:disable CodeReuse/ActiveRecord + def unique_users_all_imports(time_period) + project_imports = distinct_count(::Project.where(time_period).where.not(import_type: nil), :creator_id) + bulk_imports = distinct_count(::BulkImport.where(time_period), :user_id) + jira_issue_imports = distinct_count(::JiraImportState.where(time_period), :user_id) + csv_issue_imports = distinct_count(Issues::CsvImport.where(time_period), :user_id) + group_imports = distinct_count(::GroupImportState.where(time_period), :user_id) + + project_imports + bulk_imports + jira_issue_imports + csv_issue_imports + group_imports + end + # rubocop:enable CodeReuse/ActiveRecord + # rubocop:disable CodeReuse/ActiveRecord def distinct_count_user_auth_by_provider(time_period) counts = auth_providers_except_ldap.each_with_object({}) do |provider, hash| diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml index 4966afd534a..4d92202e7fd 100644 --- a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml @@ -4,21 +4,28 @@ # - "AND": counts unique elements that were observed triggering all of following events # events: list of events names to aggregate into metric. All events in this list must have the same 'redis_slot' and 'aggregation' attributes # see from lib/gitlab/usage_data_counters/known_events/ for the list of valid events. +# source: defines which datasource will be used to locate events that should be included in aggregated metric. Valid values are: +# - database +# - redis # feature_flag: name of development feature flag that will be checked before metrics aggregation is performed. # Corresponding feature flag should have `default_enabled` attribute set to `false`. # This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked. --- - name: compliance_features_track_unique_visits_union operator: OR + source: redis events: ['g_compliance_audit_events', 'g_compliance_dashboard', 'i_compliance_audit_events', 'a_compliance_audit_events_api', 'i_compliance_credential_inventory'] - name: product_analytics_test_metrics_union operator: OR + source: redis events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] - name: product_analytics_test_metrics_intersection operator: AND + source: redis events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] - name: incident_management_alerts_total_unique_counts operator: OR + source: redis events: [ 'incident_management_alert_status_changed', 'incident_management_alert_assigned', @@ -27,6 +34,7 @@ ] - name: incident_management_incidents_total_unique_counts operator: OR + source: redis events: [ 'incident_management_incident_created', 'incident_management_incident_reopened', @@ -40,3 +48,13 @@ 'incident_management_incident_unrelate', 'incident_management_incident_change_confidential' ] +- name: i_testing_paid_monthly_active_user_total + operator: OR + source: redis + events: [ + 'i_testing_web_performance_widget_total', + 'i_testing_full_code_quality_report_total', + 'i_testing_group_code_coverage_visit_total', + 'i_testing_load_performance_widget_total', + 'i_testing_metrics_report_widget_total' +] diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index 572ad866895..772a4623280 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -4,7 +4,9 @@ module Gitlab::UsageDataCounters class CiTemplateUniqueCounter REDIS_SLOT = 'ci_templates'.freeze + # NOTE: Events originating from implicit Auto DevOps pipelines get prefixed with `implicit_` TEMPLATE_TO_EVENT = { + '5-Minute-Production-App.gitlab-ci.yml' => '5_min_production_app', 'Auto-DevOps.gitlab-ci.yml' => 'auto_devops', 'AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml' => 'aws_cf_deploy_ec2', 'AWS/Deploy-ECS.gitlab-ci.yml' => 'aws_deploy_ecs', @@ -17,19 +19,21 @@ module Gitlab::UsageDataCounters }.freeze class << self - def track_unique_project_event(project_id:, template:) + def track_unique_project_event(project_id:, template:, config_source:) return if Feature.disabled?(:usage_data_track_ci_templates_unique_projects, default_enabled: :yaml) - if event = unique_project_event(template) + if event = unique_project_event(template, config_source) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id) end end private - def unique_project_event(template) + def unique_project_event(template, config_source) if name = TEMPLATE_TO_EVENT[template] - "p_#{REDIS_SLOT}_#{name}" + prefix = 'implicit_' if config_source.to_s == 'auto_devops_source' + + "p_#{REDIS_SLOT}_#{prefix}#{name}" end end end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index 47361d831b2..68ae239debb 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -13,15 +13,10 @@ module Gitlab AggregationMismatch = Class.new(EventError) SlotMismatch = Class.new(EventError) CategoryMismatch = Class.new(EventError) - UnknownAggregationOperator = Class.new(EventError) InvalidContext = Class.new(EventError) KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) ALLOWED_AGGREGATIONS = %i(daily weekly).freeze - UNION_OF_AGGREGATED_METRICS = 'OR' - INTERSECTION_OF_AGGREGATED_METRICS = 'AND' - ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze - AGGREGATED_METRICS_PATH = File.expand_path('aggregated_metrics/*.yml', __dir__) # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id @@ -90,37 +85,40 @@ module Gitlab events_names = events_for_category(category) event_results = events_names.each_with_object({}) do |event, hash| - hash["#{event}_weekly"] = unique_events(event_names: [event], start_date: 7.days.ago.to_date, end_date: Date.current) - hash["#{event}_monthly"] = unique_events(event_names: [event], start_date: 4.weeks.ago.to_date, end_date: Date.current) + hash["#{event}_weekly"] = unique_events(**weekly_time_range.merge(event_names: [event])) + hash["#{event}_monthly"] = unique_events(**monthly_time_range.merge(event_names: [event])) end if eligible_for_totals?(events_names) - event_results["#{category}_total_unique_counts_weekly"] = unique_events(event_names: events_names, start_date: 7.days.ago.to_date, end_date: Date.current) - event_results["#{category}_total_unique_counts_monthly"] = unique_events(event_names: events_names, start_date: 4.weeks.ago.to_date, end_date: Date.current) + event_results["#{category}_total_unique_counts_weekly"] = unique_events(**weekly_time_range.merge(event_names: events_names)) + event_results["#{category}_total_unique_counts_monthly"] = unique_events(**monthly_time_range.merge(event_names: events_names)) end category_results["#{category}"] = event_results end end - def known_event?(event_name) - event_for(event_name).present? + def weekly_time_range + { start_date: 7.days.ago.to_date, end_date: Date.current } end - def aggregated_metrics_monthly_data - aggregated_metrics_data(4.weeks.ago.to_date) + def monthly_time_range + { start_date: 4.weeks.ago.to_date, end_date: Date.current } end - def aggregated_metrics_weekly_data - aggregated_metrics_data(7.days.ago.to_date) + def known_event?(event_name) + event_for(event_name).present? end def known_events @known_events ||= load_events(KNOWN_EVENTS_PATH) end - def aggregated_metrics - @aggregated_metrics ||= load_events(AGGREGATED_METRICS_PATH) + def calculate_events_union(event_names:, start_date:, end_date:) + count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| + raise SlotMismatch, events unless events_in_same_slot?(events) + raise AggregationMismatch, events unless events_same_aggregation?(events) + end end private @@ -131,6 +129,8 @@ module Gitlab event = event_for(event_name) raise UnknownEvent, "Unknown event #{event_name}" unless event.present? + return unless feature_enabled?(event) + Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) end @@ -139,93 +139,6 @@ module Gitlab Plan.all_plans end - def aggregated_metrics_data(start_date) - aggregated_metrics.each_with_object({}) do |aggregation, weekly_data| - next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], default_enabled: false, type: :development) - - weekly_data[aggregation[:name]] = calculate_count_for_aggregation(aggregation, start_date: start_date, end_date: Date.current) - end - end - - def calculate_count_for_aggregation(aggregation, start_date:, end_date:) - case aggregation[:operator] - when UNION_OF_AGGREGATED_METRICS - calculate_events_union(event_names: aggregation[:events], start_date: start_date, end_date: end_date) - when INTERSECTION_OF_AGGREGATED_METRICS - calculate_events_intersections(event_names: aggregation[:events], start_date: start_date, end_date: end_date) - else - raise UnknownAggregationOperator, "Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}" - end - end - - # calculate intersection of 'n' sets based on inclusion exclusion principle https://en.wikipedia.org/wiki/Inclusion%E2%80%93exclusion_principle - # this method will be extracted to dedicated module with https://gitlab.com/gitlab-org/gitlab/-/issues/273391 - def calculate_events_intersections(event_names:, start_date:, end_date:, subset_powers_cache: Hash.new({})) - # calculate power of intersection of all given metrics from inclusion exclusion principle - # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C|) => - # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| - # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => - # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| - - # calculate each components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... - subset_powers_data = subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) - - # calculate last component of the equation |A & B & C & D| = .... - |A + B + C + D| - power_of_union_of_all_events = begin - subset_powers_cache[event_names.size][event_names.join('_+_')] ||= \ - calculate_events_union(event_names: event_names, start_date: start_date, end_date: end_date) - end - - # in order to determine if part of equation (|A & B & C|, |A & B & C & D|), that represents the intersection that we need to calculate, - # is positive or negative in particular equation we need to determine if number of subsets is even or odd. Please take a look at two examples below - # |A + B + C| = (|A| + |B| + |C|) - (|A & B| + |A & C| + .. + |C & D|) + |A & B & C| => - # |A & B & C| = - (|A| + |B| + |C|) + (|A & B| + |A & C| + .. + |C & D|) + |A + B + C| - # |A + B + C + D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A & B & C & D| => - # |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - |A + B + C + D| - subset_powers_size_even = subset_powers_data.size.even? - - # sum all components of equation except for the last one |A & B & C & D| = (|A| + |B| + |C| + |D|) - (|A & B| + |A & C| + .. + |C & D|) + (|A & B & C| + |B & C & D|) - ... => - sum_of_all_subset_powers = sum_subset_powers(subset_powers_data, subset_powers_size_even) - - # add last component of the equation |A & B & C & D| = sum_of_all_subset_powers - |A + B + C + D| - sum_of_all_subset_powers + (subset_powers_size_even ? power_of_union_of_all_events : -power_of_union_of_all_events) - end - - def sum_subset_powers(subset_powers_data, subset_powers_size_even) - sum_without_sign = subset_powers_data.to_enum.with_index.sum do |value, index| - (index + 1).odd? ? value : -value - end - - (subset_powers_size_even ? -1 : 1) * sum_without_sign - end - - def subsets_intersection_powers(event_names, start_date, end_date, subset_powers_cache) - subset_sizes = (1..(event_names.size - 1)) - - subset_sizes.map do |subset_size| - if subset_size > 1 - # calculate sum of powers of intersection between each subset (with given size) of metrics: #|A + B + C + D| = ... - (|A & B| + |A & C| + .. + |C & D|) - event_names.combination(subset_size).sum do |events_subset| - subset_powers_cache[subset_size][events_subset.join('_&_')] ||= \ - calculate_events_intersections(event_names: events_subset, start_date: start_date, end_date: end_date, subset_powers_cache: subset_powers_cache) - end - else - # calculate sum of powers of each set (metric) alone #|A + B + C + D| = (|A| + |B| + |C| + |D|) - ... - event_names.sum do |event| - subset_powers_cache[subset_size][event] ||= \ - unique_events(event_names: event, start_date: start_date, end_date: end_date) - end - end - end - end - - def calculate_events_union(event_names:, start_date:, end_date:) - count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| - raise SlotMismatch, events unless events_in_same_slot?(events) - raise AggregationMismatch, events unless events_same_aggregation?(events) - end - end - def count_unique_events(event_names:, start_date:, end_date:, context: '') events = events_for(Array(event_names).map(&:to_s)) @@ -237,6 +150,12 @@ module Gitlab redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end + def feature_enabled?(event) + return true if event[:feature_flag].blank? + + Feature.enabled?(event[:feature_flag], default_enabled: :yaml) + end + # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level # and if there are more than 1 event def eligible_for_totals?(events_names) @@ -340,12 +259,6 @@ module Gitlab end.flatten end - def validate_aggregation_operator!(operator) - return true if ALLOWED_METRICS_AGGREGATIONS.include?(operator) - - raise UnknownAggregationOperator.new("Events should be aggregated with one of operators #{ALLOWED_METRICS_AGGREGATIONS}") - end - def weekly_redis_keys(events:, start_date:, end_date:, context: '') end_date = end_date.end_of_week - 1.week (start_date.to_date..end_date.to_date).map do |date| diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index f649e7f407d..c2662a74432 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -145,7 +145,6 @@ module Gitlab private def track_unique_action(action, author, time) - return unless Feature.enabled?(:track_issue_activity_actions, default_enabled: true) return unless author Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml new file mode 100644 index 00000000000..9c19c9e8b8c --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -0,0 +1,91 @@ +# Implicit Auto DevOps pipeline events +- name: p_ci_templates_implicit_auto_devops + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_implicit_auto_devops_build + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_implicit_auto_devops_deploy + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_implicit_security_sast + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_implicit_security_secret_detection + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +# Explicit include:template pipeline events +- name: p_ci_templates_5_min_production_app + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_auto_devops + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_aws_cf_deploy_ec2 + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_aws_deploy_ecs + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_auto_devops_build + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_auto_devops_deploy + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_auto_devops_deploy_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_security_sast + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_security_secret_detection + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects + +- name: p_ci_templates_terraform_base_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml new file mode 100644 index 00000000000..d657c5487d7 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -0,0 +1,166 @@ +--- +- name: i_code_review_mr_diffs + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_mr_diffs +- name: i_code_review_user_single_file_diffs + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_single_file_diffs +- name: i_code_review_mr_single_file_diffs + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_mr_single_file_diffs +- name: i_code_review_user_toggled_task_item_status + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_toggled_task_item_status +- name: i_code_review_user_create_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_mr +- name: i_code_review_user_close_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_close_mr +- name: i_code_review_user_reopen_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_reopen_mr +- name: i_code_review_user_approve_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_approve_mr +- name: i_code_review_user_unapprove_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_unapprove_mr +- name: i_code_review_user_resolve_thread + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_resolve_thread +- name: i_code_review_user_unresolve_thread + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_unresolve_thread +- name: i_code_review_edit_mr_title + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_edit_mr_title +- name: i_code_review_edit_mr_desc + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_edit_mr_desc +- name: i_code_review_user_merge_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_merge_mr +- name: i_code_review_user_create_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_mr_comment +- name: i_code_review_user_edit_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_edit_mr_comment +- name: i_code_review_user_remove_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_remove_mr_comment +- name: i_code_review_user_create_review_note + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_review_note +- name: i_code_review_user_publish_review + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_publish_review +- name: i_code_review_user_create_multiline_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_multiline_mr_comment +- name: i_code_review_user_edit_multiline_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_edit_multiline_mr_comment +- name: i_code_review_user_remove_multiline_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_remove_multiline_mr_comment +- name: i_code_review_user_add_suggestion + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_add_suggestion +- name: i_code_review_user_apply_suggestion + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_apply_suggestion +- name: i_code_review_user_assigned + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_assigned +- name: i_code_review_user_marked_as_draft + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_marked_as_draft +- name: i_code_review_user_unmarked_as_draft + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_unmarked_as_draft +- name: i_code_review_user_review_requested + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_review_requested +- name: i_code_review_user_approval_rule_added + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_approval_rule_added +- name: i_code_review_user_approval_rule_deleted + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_approval_rule_deleted +- name: i_code_review_user_approval_rule_edited + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_approval_rule_edited +- name: i_code_review_user_vs_code_api_request + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_vs_code_api_request +- name: i_code_review_user_create_mr_from_issue + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_mr_from_issue diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 4cbde0c0372..79f319b2d58 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -268,172 +268,154 @@ redis_slot: testing aggregation: weekly feature_flag: usage_data_i_testing_web_performance_widget_total +- name: i_testing_group_code_coverage_project_click_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_group_code_coverage_project_click_total +- name: i_testing_load_performance_widget_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_load_performance_widget_total +- name: i_testing_metrics_report_artifact_uploaders + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_metrics_report_artifact_uploaders # Project Management group - name: g_project_management_issue_title_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_description_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_assignee_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_made_confidential category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_made_visible category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_created category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_closed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_reopened category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_label_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_milestone_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_iteration_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_weight_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_cross_referenced category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_moved category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_related category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_unrelated category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_marked_as_duplicate category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_locked category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_unlocked category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_added_to_epic category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_removed_from_epic category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_changed_epic category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_added category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_modified category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_designs_removed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_due_date_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_time_estimate_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_time_spent_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_comment_added category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_comment_edited category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_comment_removed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_health_status_changed category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions - name: g_project_management_issue_cloned category: issues_edit redis_slot: project_management aggregation: daily - feature_flag: track_issue_activity_actions # Secrets Management - name: i_ci_secrets_management_vault_build_created category: ci_secrets_management @@ -445,126 +427,15 @@ redis_slot: snippets aggregation: weekly feature_flag: usage_data_i_snippets_show -# Merge request counters -- name: i_code_review_mr_diffs - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_mr_diffs -- name: i_code_review_user_single_file_diffs - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_single_file_diffs -- name: i_code_review_mr_single_file_diffs - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_mr_single_file_diffs -- name: i_code_review_user_create_mr - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_mr -- name: i_code_review_user_close_mr - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_close_mr -- name: i_code_review_user_reopen_mr - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_reopen_mr -- name: i_code_review_user_merge_mr - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_merge_mr -- name: i_code_review_user_create_mr_comment - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_mr_comment -- name: i_code_review_user_edit_mr_comment - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_edit_mr_comment -- name: i_code_review_user_remove_mr_comment - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_remove_mr_comment -- name: i_code_review_user_create_review_note - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_review_note -- name: i_code_review_user_publish_review - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_publish_review -- name: i_code_review_user_create_multiline_mr_comment - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_create_multiline_mr_comment -- name: i_code_review_user_edit_multiline_mr_comment - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_edit_multiline_mr_comment -- name: i_code_review_user_remove_multiline_mr_comment - redis_slot: code_review - category: code_review - aggregation: weekly - feature_flag: usage_data_i_code_review_user_remove_multiline_mr_comment # Terraform - name: p_terraform_state_api_unique_users category: terraform redis_slot: terraform aggregation: weekly feature_flag: usage_data_p_terraform_state_api_unique_users -# CI templates -- name: p_ci_templates_auto_devops - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_aws_cf_deploy_ec2 - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_auto_devops_build - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_auto_devops_deploy - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_auto_devops_deploy_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_security_sast - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_security_secret_detection - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects -- name: p_ci_templates_terraform_base_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - feature_flag: usage_data_track_ci_templates_unique_projects +# Pipeline Authoring +- name: o_pipeline_authoring_unique_users_committing_ciconfigfile + category: pipeline_authoring + redis_slot: pipeline_authoring + aggregation: weekly + feature_flag: usage_data_unique_users_committing_ciconfigfile diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml new file mode 100644 index 00000000000..3fd02164f74 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -0,0 +1,22 @@ +--- +# Ecosystem category +- name: i_ecosystem_jira_service_close_issue + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_jira_service +- name: i_ecosystem_jira_service_cross_reference + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_jira_service +- name: i_ecosystem_jira_service_list_issues + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_jira_service +- name: i_ecosystem_jira_service_create_issue + category: ecosystem + redis_slot: ecosystem + aggregation: weekly + feature_flag: usage_data_track_ecosystem_jira_service diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml new file mode 100644 index 00000000000..bf292047da0 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -0,0 +1,326 @@ +--- +- name: i_quickactions_approve + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_assign_single + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_assign_multiple + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_assign_self + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_assign_reviewer + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_award + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_board_move + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_child_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_clear_weight + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_clone + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_close + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_confidential + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_copy_metadata_merge_request + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_copy_metadata_issue + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_create_merge_request + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_done + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_draft + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_due + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_duplicate + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_estimate + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_iteration + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_label + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_lock + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_merge + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_milestone + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_move + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_parent_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_promote + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_publish + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_reassign + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_reassign_reviewer + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_rebase + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_relabel + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_relate + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_child_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_due_date + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_estimate + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_iteration + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_milestone + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_parent_epic + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_time_spent + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_remove_zoom + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_reopen + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_shrug + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_spend_subtract + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_spend_add + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_submit_review + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_subscribe + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_tableflip + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_tag + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_target_branch + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_title + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_todo + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unassign_specific + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unassign_all + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unassign_reviewer + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unlabel_specific + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unlabel_all + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unlock + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_unsubscribe + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_weight + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_wip + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions +- name: i_quickactions_zoom + category: quickactions + redis_slot: quickactions + aggregation: weekly + feature_flag: usage_data_track_quickactions diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 11d59257ed9..b9856e1f74a 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -10,6 +10,8 @@ module Gitlab MR_CLOSE_ACTION = 'i_code_review_user_close_mr' MR_REOPEN_ACTION = 'i_code_review_user_reopen_mr' MR_MERGE_ACTION = 'i_code_review_user_merge_mr' + MR_APPROVE_ACTION = 'i_code_review_user_approve_mr' + MR_UNAPPROVE_ACTION = 'i_code_review_user_unapprove_mr' MR_CREATE_COMMENT_ACTION = 'i_code_review_user_create_mr_comment' MR_EDIT_COMMENT_ACTION = 'i_code_review_user_edit_mr_comment' MR_REMOVE_COMMENT_ACTION = 'i_code_review_user_remove_mr_comment' @@ -18,6 +20,21 @@ module Gitlab MR_CREATE_MULTILINE_COMMENT_ACTION = 'i_code_review_user_create_multiline_mr_comment' MR_EDIT_MULTILINE_COMMENT_ACTION = 'i_code_review_user_edit_multiline_mr_comment' MR_REMOVE_MULTILINE_COMMENT_ACTION = 'i_code_review_user_remove_multiline_mr_comment' + MR_ADD_SUGGESTION_ACTION = 'i_code_review_user_add_suggestion' + MR_APPLY_SUGGESTION_ACTION = 'i_code_review_user_apply_suggestion' + MR_MARKED_AS_DRAFT_ACTION = 'i_code_review_user_marked_as_draft' + MR_UNMARKED_AS_DRAFT_ACTION = 'i_code_review_user_unmarked_as_draft' + MR_RESOLVE_THREAD_ACTION = 'i_code_review_user_resolve_thread' + MR_UNRESOLVE_THREAD_ACTION = 'i_code_review_user_unresolve_thread' + MR_ASSIGNED_USERS_ACTION = 'i_code_review_user_assigned' + MR_REVIEW_REQUESTED_USERS_ACTION = 'i_code_review_user_review_requested' + MR_TASK_ITEM_STATUS_CHANGED_ACTION = 'i_code_review_user_toggled_task_item_status' + MR_APPROVAL_RULE_ADDED_USERS_ACTION = 'i_code_review_user_approval_rule_added' + MR_APPROVAL_RULE_EDITED_USERS_ACTION = 'i_code_review_user_approval_rule_edited' + MR_APPROVAL_RULE_DELETED_USERS_ACTION = 'i_code_review_user_approval_rule_deleted' + MR_EDIT_MR_TITLE_ACTION = 'i_code_review_edit_mr_title' + MR_EDIT_MR_DESC_ACTION = 'i_code_review_edit_mr_desc' + MR_CREATE_FROM_ISSUE_ACTION = 'i_code_review_user_create_mr_from_issue' class << self def track_mr_diffs_action(merge_request:) @@ -45,6 +62,22 @@ module Gitlab track_unique_action_by_user(MR_REOPEN_ACTION, user) end + def track_approve_mr_action(user:) + track_unique_action_by_user(MR_APPROVE_ACTION, user) + end + + def track_unapprove_mr_action(user:) + track_unique_action_by_user(MR_UNAPPROVE_ACTION, user) + end + + def track_resolve_thread_action(user:) + track_unique_action_by_user(MR_RESOLVE_THREAD_ACTION, user) + end + + def track_unresolve_thread_action(user:) + track_unique_action_by_user(MR_UNRESOLVE_THREAD_ACTION, user) + end + def track_create_comment_action(note:) track_unique_action_by_user(MR_CREATE_COMMENT_ACTION, note.author) track_multiline_unique_action(MR_CREATE_MULTILINE_COMMENT_ACTION, note) @@ -68,6 +101,58 @@ module Gitlab track_unique_action_by_user(MR_PUBLISH_REVIEW_ACTION, user) end + def track_add_suggestion_action(user:) + track_unique_action_by_user(MR_ADD_SUGGESTION_ACTION, user) + end + + def track_marked_as_draft_action(user:) + track_unique_action_by_user(MR_MARKED_AS_DRAFT_ACTION, user) + end + + def track_unmarked_as_draft_action(user:) + track_unique_action_by_user(MR_UNMARKED_AS_DRAFT_ACTION, user) + end + + def track_apply_suggestion_action(user:) + track_unique_action_by_user(MR_APPLY_SUGGESTION_ACTION, user) + end + + def track_users_assigned_to_mr(users:) + track_unique_action_by_users(MR_ASSIGNED_USERS_ACTION, users) + end + + def track_users_review_requested(users:) + track_unique_action_by_users(MR_REVIEW_REQUESTED_USERS_ACTION, users) + end + + def track_title_edit_action(user:) + track_unique_action_by_user(MR_EDIT_MR_TITLE_ACTION, user) + end + + def track_description_edit_action(user:) + track_unique_action_by_user(MR_EDIT_MR_DESC_ACTION, user) + end + + def track_approval_rule_added_action(user:) + track_unique_action_by_user(MR_APPROVAL_RULE_ADDED_USERS_ACTION, user) + end + + def track_approval_rule_edited_action(user:) + track_unique_action_by_user(MR_APPROVAL_RULE_EDITED_USERS_ACTION, user) + end + + def track_approval_rule_deleted_action(user:) + track_unique_action_by_user(MR_APPROVAL_RULE_DELETED_USERS_ACTION, user) + end + + def track_task_item_status_changed(user:) + track_unique_action_by_user(MR_TASK_ITEM_STATUS_CHANGED_ACTION, user) + end + + def track_mr_create_from_issue(user:) + track_unique_action_by_user(MR_CREATE_FROM_ISSUE_ACTION, user) + end + private def track_unique_action_by_merge_request(action, merge_request) @@ -80,6 +165,12 @@ module Gitlab track_unique_action(action, user.id) end + def track_unique_action_by_users(action, users) + return if users.blank? + + track_unique_action(action, users.map(&:id)) + end + def track_unique_action(action, value) Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) end diff --git a/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb new file mode 100644 index 00000000000..f757b51f73c --- /dev/null +++ b/lib/gitlab/usage_data_counters/quick_action_activity_unique_counter.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module QuickActionActivityUniqueCounter + class << self + # Tracks the quick action with name `name`. + # `args` is expected to be a single string, will be split internally when necessary. + def track_unique_action(name, args:, user:) + return unless Feature.enabled?(:usage_data_track_quickactions, default_enabled: :yaml) + return unless user + + args ||= '' + name = prepare_name(name, args) + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(:"i_quickactions_#{name}", values: user.id) + end + + private + + def prepare_name(name, args) + case name + when 'assign' + event_name_for_assign(args) + when 'copy_metadata' + event_name_for_copy_metadata(args) + when 'remove_reviewer' + 'unassign_reviewer' + when 'request_review', 'reviewer' + 'assign_reviewer' + when 'spend' + event_name_for_spend(args) + when 'unassign' + event_name_for_unassign(args) + when 'unlabel', 'remove_label' + event_name_for_unlabel(args) + else + name + end + end + + def event_name_for_assign(args) + args = args.split + + if args.count == 1 && args.first == 'me' + 'assign_self' + elsif args.count == 1 + 'assign_single' + else + 'assign_multiple' + end + end + + def event_name_for_copy_metadata(args) + if args.start_with?('#') + 'copy_metadata_issue' + else + 'copy_metadata_merge_request' + end + end + + def event_name_for_spend(args) + if args.start_with?('-') + 'spend_subtract' + else + 'spend_add' + end + end + + def event_name_for_unassign(args) + if args.present? + 'unassign_specific' + else + 'unassign_all' + end + end + + def event_name_for_unlabel(args) + if args.present? + 'unlabel_specific' + else + 'unlabel_all' + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/vs_code_extension_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/vs_code_extension_activity_unique_counter.rb new file mode 100644 index 00000000000..703c4885b04 --- /dev/null +++ b/lib/gitlab/usage_data_counters/vs_code_extension_activity_unique_counter.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module VSCodeExtensionActivityUniqueCounter + VS_CODE_API_REQUEST_ACTION = 'i_code_review_user_vs_code_api_request' + VS_CODE_USER_AGENT_REGEX = /\Avs-code-gitlab-workflow/.freeze + + class << self + def track_api_request_when_trackable(user_agent:, user:) + user_agent&.match?(VS_CODE_USER_AGENT_REGEX) && track_unique_action_by_user(VS_CODE_API_REQUEST_ACTION, user) + end + + private + + def track_unique_action_by_user(action, user) + return unless user + + track_unique_action(action, user.id) + end + + def track_unique_action(action, value) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) + end + end + end + end +end diff --git a/lib/gitlab/utils/markdown.rb b/lib/gitlab/utils/markdown.rb index e783ac785cc..5087020affe 100644 --- a/lib/gitlab/utils/markdown.rb +++ b/lib/gitlab/utils/markdown.rb @@ -4,7 +4,7 @@ module Gitlab module Utils module Markdown PUNCTUATION_REGEXP = /[^\p{Word}\- ]/u.freeze - PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate)(\s+only)?\)\**/.freeze + PRODUCT_SUFFIX = /\s*\**\((core|starter|premium|ultimate|free|bronze|silver|gold)(\s+(only|self|sass))?\)\**/.freeze def string_to_anchor(string) string diff --git a/lib/gitlab/utils/measuring.rb b/lib/gitlab/utils/measuring.rb index febe489f1f8..ffd12c1b518 100644 --- a/lib/gitlab/utils/measuring.rb +++ b/lib/gitlab/utils/measuring.rb @@ -53,14 +53,10 @@ module Gitlab end def with_gc_stats - GC.start # perform a full mark-and-sweep - stats_before = GC.stat + stats = ::Gitlab::Memory::Instrumentation.start_thread_memory_allocations yield - stats_after = GC.stat - @gc_stats = stats_after.map do |key, after_value| - before_value = stats_before[key] - [key, before: before_value, after: after_value, diff: after_value - before_value] - end.to_h + ensure + @gc_stats = ::Gitlab::Memory::Instrumentation.measure_thread_memory_allocations(stats) end def with_measure_time diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 784a6686962..c92865636d0 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -153,7 +153,13 @@ module Gitlab def extended(mod = nil) super - queue_verification(mod.singleton_class) if mod + # Hack to resolve https://gitlab.com/gitlab-org/gitlab/-/issues/23932 + is_not_concern_hack = + (mod.is_a?(Class) || !name&.end_with?('::ClassMethods')) + + if mod && is_not_concern_hack + queue_verification(mod.singleton_class) + end end def queue_verification(base, verify: false) @@ -174,7 +180,7 @@ module Gitlab end def self.verify! - extensions.values.each(&:verify!) + extensions.each_value(&:verify!) end end end diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index baccadd9594..28dc66e19f8 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -39,6 +39,9 @@ module Gitlab FALLBACK = -1 DISTRIBUTED_HLL_FALLBACK = -2 + ALL_TIME_PERIOD_HUMAN_NAME = "all_time" + WEEKLY_PERIOD_HUMAN_NAME = "weekly" + MONTHLY_PERIOD_HUMAN_NAME = "monthly" def count(relation, column = nil, batch: true, batch_size: nil, start: nil, finish: nil) if batch @@ -61,10 +64,13 @@ module Gitlab end def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) - Gitlab::Database::PostgresHll::BatchDistinctCounter + buckets = Gitlab::Database::PostgresHll::BatchDistinctCounter .new(relation, column) .execute(batch_size: batch_size, start: start, finish: finish) - .estimated_distinct_count + + yield buckets if block_given? + + buckets.estimated_distinct_count rescue ActiveRecord::StatementInvalid FALLBACK # catch all rescue should be removed as a part of feature flag rollout issue diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 8e7af8876a4..e9905bae985 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -32,7 +32,7 @@ module Gitlab GitalyServer: { address: Gitlab::GitalyClient.address(repository.storage), token: Gitlab::GitalyClient.token(repository.storage), - features: Feature::Gitaly.server_feature_flags + features: Feature::Gitaly.server_feature_flags(repository.project) } } @@ -231,7 +231,7 @@ module Gitlab { address: Gitlab::GitalyClient.address(repository.shard), token: Gitlab::GitalyClient.token(repository.shard), - features: Feature::Gitaly.server_feature_flags + features: Feature::Gitaly.server_feature_flags(repository.project) } end |