diff options
Diffstat (limited to 'lib/gitlab')
183 files changed, 3408 insertions, 1717 deletions
diff --git a/lib/gitlab/analytics/cycle_analytics/default_stages.rb b/lib/gitlab/analytics/cycle_analytics/default_stages.rb index 22aa680cbc1..43683ae174e 100644 --- a/lib/gitlab/analytics/cycle_analytics/default_stages.rb +++ b/lib/gitlab/analytics/cycle_analytics/default_stages.rb @@ -30,6 +30,10 @@ module Gitlab all.map { |stage| stage[:name] } end + def self.symbolized_stage_names + names.map(&:to_sym) + end + def self.params_for_issue_stage { name: 'issue', diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events.rb b/lib/gitlab/analytics/cycle_analytics/stage_events.rb index 39dc706dff5..27fc8bd9a1a 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events.rb @@ -11,6 +11,7 @@ module Gitlab ENUM_MAPPING = { StageEvents::IssueCreated => 1, StageEvents::IssueFirstMentionedInCommit => 2, + StageEvents::IssueDeployedToProduction => 3, StageEvents::MergeRequestCreated => 100, StageEvents::MergeRequestFirstDeployedToProduction => 101, StageEvents::MergeRequestLastBuildFinished => 102, @@ -18,8 +19,7 @@ module Gitlab StageEvents::MergeRequestMerged => 104, StageEvents::CodeStageStart => 1_000, StageEvents::IssueStageEnd => 1_001, - StageEvents::PlanStageStart => 1_002, - StageEvents::ProductionStageEnd => 1_003 + StageEvents::PlanStageStart => 1_002 }.freeze EVENTS = ENUM_MAPPING.keys.freeze @@ -27,8 +27,7 @@ module Gitlab INTERNAL_EVENTS = [ StageEvents::CodeStageStart, StageEvents::IssueStageEnd, - StageEvents::PlanStageStart, - StageEvents::ProductionStageEnd + StageEvents::PlanStageStart ].freeze # Defines which start_event and end_event pairs are allowed @@ -41,7 +40,7 @@ module Gitlab ], StageEvents::IssueCreated => [ StageEvents::IssueStageEnd, - StageEvents::ProductionStageEnd + StageEvents::IssueDeployedToProduction ], StageEvents::MergeRequestCreated => [ StageEvents::MergeRequestMerged diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb index b778364a917..3e93e60e686 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/production_stage_end.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb @@ -4,13 +4,13 @@ module Gitlab module Analytics module CycleAnalytics module StageEvents - class ProductionStageEnd < StageEvent + class IssueDeployedToProduction < StageEvent def self.name _("Issue first deployed to production") end def self.identifier - :production_stage_end + :issue_deployed_to_production end def object_type diff --git a/lib/gitlab/analytics/unique_visits.rb b/lib/gitlab/analytics/unique_visits.rb index 292048dcad9..e367d33d743 100644 --- a/lib/gitlab/analytics/unique_visits.rb +++ b/lib/gitlab/analytics/unique_visits.rb @@ -4,7 +4,7 @@ module Gitlab module Analytics class UniqueVisits def track_visit(visitor_id, target_id, time = Time.zone.now) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(visitor_id, target_id, time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(target_id, values: visitor_id, time: time) end # Returns number of unique visitors for given targets in given time frame diff --git a/lib/gitlab/api_authentication/builder.rb b/lib/gitlab/api_authentication/builder.rb new file mode 100644 index 00000000000..717c664826a --- /dev/null +++ b/lib/gitlab/api_authentication/builder.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# Authentication Strategies Builder +# +# AuthBuilder and its child classes, TokenType and SentThrough, support +# declaring allowed authentication strategies with patterns like +# `accept.token_type(:job_token).sent_through(:http_basic)`. +module Gitlab + module APIAuthentication + class Builder + def build + strategies = Hash.new([]) + yield ::Gitlab::APIAuthentication::TokenTypeBuilder.new(strategies) + strategies + end + end + end +end diff --git a/lib/gitlab/api_authentication/sent_through_builder.rb b/lib/gitlab/api_authentication/sent_through_builder.rb new file mode 100644 index 00000000000..f66e5960019 --- /dev/null +++ b/lib/gitlab/api_authentication/sent_through_builder.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# See Gitlab::APIAuthentication::Builder +module Gitlab + module APIAuthentication + class SentThroughBuilder + def initialize(strategies, resolvers) + @strategies = strategies + @resolvers = resolvers + end + + def sent_through(*locators) + locators.each do |locator| + @strategies[locator] |= @resolvers + end + end + end + end +end diff --git a/lib/gitlab/api_authentication/token_locator.rb b/lib/gitlab/api_authentication/token_locator.rb new file mode 100644 index 00000000000..32a98908e5b --- /dev/null +++ b/lib/gitlab/api_authentication/token_locator.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module APIAuthentication + class TokenLocator + UsernameAndPassword = Struct.new(:username, :password) + + include ActiveModel::Validations + include ActionController::HttpAuthentication::Basic + + attr_reader :location + + validates :location, inclusion: { in: %i[http_basic_auth] } + + def initialize(location) + @location = location + validate! + end + + def extract(request) + case @location + when :http_basic_auth + extract_from_http_basic_auth request + end + end + + private + + def extract_from_http_basic_auth(request) + username, password = user_name_and_password(request) + return unless username.present? && password.present? + + UsernameAndPassword.new(username, password) + end + end + end +end diff --git a/lib/gitlab/api_authentication/token_resolver.rb b/lib/gitlab/api_authentication/token_resolver.rb new file mode 100644 index 00000000000..5b30777b6ec --- /dev/null +++ b/lib/gitlab/api_authentication/token_resolver.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module APIAuthentication + class TokenResolver + include ActiveModel::Validations + + attr_reader :token_type + + validates :token_type, inclusion: { in: %i[personal_access_token job_token deploy_token] } + + def initialize(token_type) + @token_type = token_type + validate! + end + + # Existing behavior is known to be inconsistent across authentication + # methods with regards to whether to silently ignore present but invalid + # credentials or to raise an error/respond with 401. + # + # If a token can be located from the provided credentials, but the token + # or credentials are in some way invalid, this implementation opts to + # raise an error. + # + # For example, if the raw credentials include a username and password, and + # a token is resolved from the password, but the username does not match + # the token, an error will be raised. + # + # See https://gitlab.com/gitlab-org/gitlab/-/issues/246569 + + def resolve(raw) + case @token_type + when :personal_access_token + resolve_personal_access_token raw + + when :job_token + resolve_job_token raw + + when :deploy_token + resolve_deploy_token raw + end + end + + private + + def resolve_personal_access_token(raw) + # Check if the password is a personal access token + pat = ::PersonalAccessToken.find_by_token(raw.password) + return unless pat + + # Ensure that the username matches the token. This check is a subtle + # departure from the existing behavior of #find_personal_access_token_from_http_basic_auth. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_435907856 + raise ::Gitlab::Auth::UnauthorizedError unless pat.user.username == raw.username + + pat + end + + def resolve_job_token(raw) + # Only look for a job if the username is correct + return if ::Gitlab::Auth::CI_JOB_USER != raw.username + + job = ::Ci::AuthJobFinder.new(token: raw.password).execute + + # Actively reject credentials with the username `gitlab-ci-token` if + # the password is not a valid job token. This replicates existing + # behavior of #find_user_from_job_token. + raise ::Gitlab::Auth::UnauthorizedError unless job + + job + end + + def resolve_deploy_token(raw) + # Check if the password is a deploy token + token = ::DeployToken.active.find_by_token(raw.password) + return unless token + + # Ensure that the username matches the token. This check is a subtle + # departure from the existing behavior of #deploy_token_from_request. + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/38627#note_474826205 + raise ::Gitlab::Auth::UnauthorizedError unless token.username == raw.username + + token + end + end + end +end diff --git a/lib/gitlab/api_authentication/token_type_builder.rb b/lib/gitlab/api_authentication/token_type_builder.rb new file mode 100644 index 00000000000..4a57cdc2742 --- /dev/null +++ b/lib/gitlab/api_authentication/token_type_builder.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# See Gitlab::Auth::AuthBuilder +module Gitlab + module APIAuthentication + class TokenTypeBuilder + def initialize(strategies) + @strategies = strategies + end + + def token_types(*resolvers) + ::Gitlab::APIAuthentication::SentThroughBuilder.new(@strategies, resolvers) + end + + alias_method :token_type, :token_types + end + end +end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 84fe3d1c959..cefe983848c 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -12,6 +12,7 @@ module Gitlab Attribute.new(:namespace, Namespace), Attribute.new(:user, User), Attribute.new(:caller_id, String), + Attribute.new(:remote_ip, String), Attribute.new(:related_class, String), Attribute.new(:feature_category, String) ].freeze @@ -45,6 +46,7 @@ module Gitlab hash[:project] = -> { project_path } if set_values.include?(:project) hash[:root_namespace] = -> { root_namespace_path } if include_namespace? hash[:caller_id] = caller_id if set_values.include?(:caller_id) + hash[:remote_ip] = remote_ip if set_values.include?(:remote_ip) hash[:related_class] = related_class if set_values.include?(:related_class) hash[:feature_category] = feature_category if set_values.include?(:feature_category) end diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index caa881eeeab..4c6254c9e69 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -92,10 +92,10 @@ module Gitlab # We only allow Private Access Tokens with `api` scope to be used by web # requests on RSS feeds or ICS files for backwards compatibility. # It is also used by GraphQL/API requests. - def find_user_from_web_access_token(request_format) + def find_user_from_web_access_token(request_format, scopes: [:api]) return unless access_token && valid_web_access_format?(request_format) - validate_access_token!(scopes: [:api]) + validate_access_token!(scopes: scopes) ::PersonalAccessTokens::LastUsedService.new(access_token).execute @@ -194,11 +194,15 @@ module Gitlab def access_token strong_memoize(:access_token) do - # The token can be a PAT or an OAuth (doorkeeper) token - # It is also possible that a PAT is encapsulated in a `Bearer` OAuth token - # (e.g. NPM client registry auth), this case will be properly handled - # by find_personal_access_token - find_oauth_access_token || find_personal_access_token + if try(:namespace_inheritable, :authentication) + access_token_from_namespace_inheritable + else + # The token can be a PAT or an OAuth (doorkeeper) token + # It is also possible that a PAT is encapsulated in a `Bearer` OAuth token + # (e.g. NPM client registry auth), this case will be properly handled + # by find_personal_access_token + find_oauth_access_token || find_personal_access_token + end end end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index f5931a1d5eb..97e4f921228 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -28,7 +28,7 @@ module Gitlab end def self.servers - Gitlab.config.ldap['servers']&.values || [] + Gitlab.config.ldap.servers&.values || [] end def self.available_servers @@ -42,9 +42,18 @@ module Gitlab end def self.providers - servers.map { |server| server['provider_name'] } + provider_names_from_servers(servers) end + def self.available_providers + provider_names_from_servers(available_servers) + end + + def self.provider_names_from_servers(servers) + servers&.map { |server| server['provider_name'] } || [] + end + private_class_method :provider_names_from_servers + def self.valid_provider?(provider) providers.include?(provider) end diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index d28ee54cfbc..504265a83ef 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -30,7 +30,7 @@ module Gitlab end def find_sessionless_user(request_format) - find_user_from_web_access_token(request_format) || + find_user_from_web_access_token(request_format, scopes: [:api, :read_api]) || find_user_from_feed_token(request_format) || find_user_from_static_object_token(request_format) || find_user_from_basic_auth_job || diff --git a/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb b/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb new file mode 100644 index 00000000000..0a8c203421b --- /dev/null +++ b/lib/gitlab/background_migration/backfill_artifact_expiry_date.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill expire_at for a range of Ci::JobArtifact + class BackfillArtifactExpiryDate + include Gitlab::Utils::StrongMemoize + + BATCH_SIZE = 1_000 + DEFAULT_EXPIRATION_SWITCH_DATE = Date.new(2020, 6, 22).freeze + OLD_ARTIFACT_AGE = 15.months + OLD_ARTIFACT_EXPIRY_OFFSET = 3.months + RECENT_ARTIFACT_EXPIRY_OFFSET = 1.year + + # Ci::JobArtifact model + class Ci::JobArtifact < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'ci_job_artifacts' + + scope :between, -> (start_id, end_id) { where(id: start_id..end_id) } + scope :before_default_expiration_switch, -> { where('created_at < ?', DEFAULT_EXPIRATION_SWITCH_DATE) } + scope :without_expiry_date, -> { where(expire_at: nil) } + scope :old, -> { where(self.arel_table[:created_at].lt(OLD_ARTIFACT_AGE.ago)) } + scope :recent, -> { where(self.arel_table[:created_at].gt(OLD_ARTIFACT_AGE.ago)) } + end + + def perform(start_id, end_id) + Ci::JobArtifact.between(start_id, end_id) + .without_expiry_date.before_default_expiration_switch + .each_batch(of: BATCH_SIZE) do |batch| + batch.old.update_all(expire_at: old_artifact_expiry_date) + batch.recent.update_all(expire_at: recent_artifact_expiry_date) + end + end + + private + + def offset_date + strong_memoize(:offset_date) do + current_date = Time.current + target_date = Time.zone.local(current_date.year, current_date.month, 22, 0, 0, 0) + + current_date.day < 22 ? target_date : target_date.next_month + end + end + + def old_artifact_expiry_date + offset_date + OLD_ARTIFACT_EXPIRY_OFFSET + end + + def recent_artifact_expiry_date + offset_date + RECENT_ARTIFACT_EXPIRY_OFFSET + end + end + end +end diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb new file mode 100644 index 00000000000..16c0de39a3b --- /dev/null +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration that extends CopyColumn to update the value of a + # column using the value of another column in the same table. + # + # - The {start_id, end_id} arguments are at the start so that it can be used + # with `queue_background_migration_jobs_by_range_at_intervals` + # - Provides support for background job tracking through the use of + # Gitlab::Database::BackgroundMigrationJob + # - Uses sub-batching so that we can keep each update's execution time at + # low 100s ms, while being able to update more records per 2 minutes + # that we allow background migration jobs to be scheduled one after the other + # - We skip the NULL checks as they may result in not using an index scan + # - The table that is migrated does _not_ need `id` as the primary key + # We use the provided primary_key column to perform the update. + class CopyColumnUsingBackgroundMigrationJob + include Gitlab::Database::DynamicModelHelpers + + PAUSE_SECONDS = 0.1 + + # start_id - The start ID of the range of rows to update. + # end_id - The end ID of the range of rows to update. + # table - The name of the table that contains the columns. + # primary_key - The primary key column of the table. + # copy_from - The column containing the data to copy. + # copy_to - The column to copy the data to. + # sub_batch_size - We don't want updates to take more than ~100ms + # This allows us to run multiple smaller batches during + # the minimum 2.minute interval that we can schedule jobs + def perform(start_id, end_id, table, primary_key, copy_from, copy_to, sub_batch_size) + quoted_copy_from = connection.quote_column_name(copy_from) + quoted_copy_to = connection.quote_column_name(copy_to) + + parent_batch_relation = relation_scoped_to_range(table, primary_key, start_id, end_id) + + parent_batch_relation.each_batch(column: primary_key, of: sub_batch_size) do |sub_batch| + sub_batch.update_all("#{quoted_copy_to}=#{quoted_copy_from}") + + sleep(PAUSE_SECONDS) + end + + # We have to add all arguments when marking a job as succeeded as they + # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` + mark_job_as_succeeded(start_id, end_id, table, primary_key, copy_from, copy_to, sub_batch_size) + end + + private + + def connection + ActiveRecord::Base.connection + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments) + end + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table).where(source_key_column => start_id..stop_id) + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb new file mode 100644 index 00000000000..52b09e07fd5 --- /dev/null +++ b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class populates the `finding_uuid` attribute for + # the existing `vulnerability_feedback` records. + class PopulateFindingUuidForVulnerabilityFeedback + REPORT_TYPES = { + sast: 0, + dependency_scanning: 1, + container_scanning: 2, + dast: 3, + secret_detection: 4, + coverage_fuzzing: 5, + api_fuzzing: 6 + }.freeze + + class VulnerabilityFeedback < ActiveRecord::Base # rubocop:disable Style/Documentation + include EachBatch + + self.table_name = 'vulnerability_feedback' + + enum category: REPORT_TYPES + + scope :in_range, -> (start, stop) { where(id: start..stop) } + scope :without_uuid, -> { where(finding_uuid: nil) } + + def self.load_vulnerability_findings + all.to_a.tap { |collection| collection.each(&:vulnerability_finding) } + end + + def set_finding_uuid + return unless vulnerability_finding.present? && vulnerability_finding.primary_identifier.present? + + update_column(:finding_uuid, calculated_uuid) + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + end + + def vulnerability_finding + BatchLoader.for(finding_key).batch(replace_methods: false) do |finding_keys, loader| + project_ids = finding_keys.map { |key| key[:project_id] } + categories = finding_keys.map { |key| key[:category] } + fingerprints = finding_keys.map { |key| key[:project_fingerprint] } + + findings = Finding.with_primary_identifier.where( + project_id: project_ids.uniq, + report_type: categories.uniq, + project_fingerprint: fingerprints.uniq + ).to_a + + finding_keys.each do |finding_key| + loader.call( + finding_key, + findings.find { |f| finding_key == f.finding_key } + ) + end + end + end + + private + + def calculated_uuid + Gitlab::UUID.v5(uuid_components) + end + + def uuid_components + [ + category, + vulnerability_finding.primary_identifier.fingerprint, + vulnerability_finding.location_fingerprint, + project_id + ].join('-') + end + + def finding_key + { + project_id: project_id, + category: category, + project_fingerprint: project_fingerprint + } + end + end + + class Finding < ActiveRecord::Base # rubocop:disable Style/Documentation + include ShaAttribute + + self.table_name = 'vulnerability_occurrences' + + sha_attribute :project_fingerprint + sha_attribute :location_fingerprint + + belongs_to :primary_identifier, class_name: 'Gitlab::BackgroundMigration::PopulateFindingUuidForVulnerabilityFeedback::Identifier' + + enum report_type: REPORT_TYPES + + scope :with_primary_identifier, -> { includes(:primary_identifier) } + + def finding_key + { + project_id: project_id, + category: report_type, + project_fingerprint: project_fingerprint + } + end + end + + class Identifier < ActiveRecord::Base # rubocop:disable Style/Documentation + self.table_name = 'vulnerability_identifiers' + end + + def perform(*range) + feedback = VulnerabilityFeedback.without_uuid.in_range(*range).load_vulnerability_findings + feedback.each(&:set_finding_uuid) + + log_info(feedback.count) + end + + def log_info(feedback_count) + ::Gitlab::BackgroundMigration::Logger.info( + migrator: self.class.name, + message: '`finding_uuid` attributes has been set', + count: feedback_count + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_duplicate_services.rb b/lib/gitlab/background_migration/remove_duplicate_services.rb new file mode 100644 index 00000000000..59fb9143a72 --- /dev/null +++ b/lib/gitlab/background_migration/remove_duplicate_services.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Remove duplicated service records with the same project and type. + # These were created in the past for unknown reasons, and should be blocked + # now by the uniqueness validation in the Service model. + class RemoveDuplicateServices + # See app/models/service + class Service < ActiveRecord::Base + include EachBatch + + self.table_name = 'services' + self.inheritance_column = :_type_disabled + + scope :project_ids_with_duplicates, -> do + select(:project_id) + .distinct + .where.not(project_id: nil) + .group(:project_id, :type) + .having('count(*) > 1') + end + + scope :types_with_duplicates, -> (project_ids) do + select(:project_id, :type) + .where(project_id: project_ids) + .group(:project_id, :type) + .having('count(*) > 1') + end + end + + def perform(*project_ids) + types_with_duplicates = Service.types_with_duplicates(project_ids).pluck(:project_id, :type) + + types_with_duplicates.each do |project_id, type| + remove_duplicates(project_id, type) + end + end + + private + + def remove_duplicates(project_id, type) + scope = Service.where(project_id: project_id, type: type) + + # Build a subquery to determine which service record is actually in use, + # by querying for it without specifying an order. + # + # This should match the record returned by `Project#find_service`, + # and the `has_one` service associations on `Project`. + correct_service = scope.select(:id).limit(1) + + # Delete all other services with the same `project_id` and `type` + duplicate_services = scope.where.not(id: correct_service) + duplicate_services.delete_all + end + end + end +end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index c0b228dee59..b146fea66b9 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -6,37 +6,20 @@ module Gitlab include Gitlab::Utils::StrongMemoize LOG_MESSAGES = { - validate_file_paths: "Validating diffs' file paths...", - diff_content_check: "Validating diff contents..." + validate_file_paths: "Validating diffs' file paths..." }.freeze def validate! return if deletion? - return unless should_run_diff_validations? + return unless should_run_validations? return if commits.empty? - file_paths = [] - - if ::Feature.enabled?(:diff_check_with_paths_changed_rpc, project, default_enabled: true) - paths = project.repository.find_changed_paths(commits.map(&:sha)) - paths.each do |path| - file_paths.concat([path.path]) - - validate_diff(path) - end - else - process_commits do |commit| - validate_once(commit) do - commit.raw_deltas.each do |diff| - file_paths.concat([diff.new_path, diff.old_path].compact) - - validate_diff(diff) - end - end - end + paths = project.repository.find_changed_paths(commits.map(&:sha)) + paths.each do |path| + validate_path(path) end - validate_file_paths(file_paths.uniq) + validate_file_paths(paths.map(&:path).uniq) end private @@ -47,43 +30,30 @@ module Gitlab end end - def should_run_diff_validations? - validations_for_diff.present? || path_validations.present? + def should_run_validations? + validations_for_path.present? || file_paths_validations.present? end - def validate_diff(diff) - validations_for_diff.each do |validation| - if error = validation.call(diff) + def validate_path(path) + validations_for_path.each do |validation| + if error = validation.call(path) raise ::Gitlab::GitAccess::ForbiddenError, error end end end # Method overwritten in EE to inject custom validations - def validations_for_diff + def validations_for_path [] end - def path_validations + def file_paths_validations validate_lfs_file_locks? ? [lfs_file_locks_validation] : [] end - def process_commits - logger.log_timed(LOG_MESSAGES[:diff_content_check]) do - # n+1: https://gitlab.com/gitlab-org/gitlab/issues/3593 - ::Gitlab::GitalyClient.allow_n_plus_1_calls do - commits.each do |commit| - logger.check_timeout_reached - - yield(commit) - end - end - end - end - def validate_file_paths(file_paths) logger.log_timed(LOG_MESSAGES[__method__]) do - path_validations.each do |validation| + file_paths_validations.each do |validation| if error = validation.call(file_paths) raise ::Gitlab::GitAccess::ForbiddenError, error end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 071a8ef830f..8ed4dc61920 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -70,6 +70,10 @@ module Gitlab @normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs end + def included_templates + @context.expandset.filter_map { |i| i[:template] } + end + private def expand_config(config) @@ -98,7 +102,8 @@ module Gitlab project: project, sha: sha || project&.repository&.root_ref_sha, user: user, - parent_pipeline: parent_pipeline) + parent_pipeline: parent_pipeline, + variables: project&.predefined_variables&.to_runner_variables) end def track_and_raise_for_dev_exception(error) diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb index 206dbaea272..6118ff49928 100644 --- a/lib/gitlab/ci/config/entry/artifacts.rb +++ b/lib/gitlab/ci/config/entry/artifacts.rb @@ -12,7 +12,7 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude].freeze + ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude public].freeze EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces" @@ -27,6 +27,7 @@ module Gitlab with_options allow_nil: true do validates :name, type: String + validates :public, boolean: true validates :untracked, boolean: true validates :paths, array_of_strings: true validates :paths, array_of_strings: { diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index cf6c2961ee7..e0adb1b19c2 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -7,14 +7,15 @@ module Gitlab class Context TimeoutError = Class.new(StandardError) - attr_reader :project, :sha, :user, :parent_pipeline + attr_reader :project, :sha, :user, :parent_pipeline, :variables attr_reader :expandset, :execution_deadline - def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil) + def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: []) @project = project @sha = sha @user = user @parent_pipeline = parent_pipeline + @variables = variables @expandset = Set.new @execution_deadline = 0 diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index e74f5b33de7..fdb3e1b00f9 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -41,7 +41,8 @@ module Gitlab project: context.project, sha: context.sha, user: context.user, - parent_pipeline: context.parent_pipeline + parent_pipeline: context.parent_pipeline, + variables: context.variables } end end diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index be479741784..114d493381c 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -72,7 +72,8 @@ module Gitlab project: project, sha: sha, user: context.user, - parent_pipeline: context.parent_pipeline + parent_pipeline: context.parent_pipeline, + variables: context.variables } end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 90692eafc3f..4d91cfd4c57 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -34,6 +34,7 @@ module Gitlab .compact .map(&method(:normalize_location)) .flat_map(&method(:expand_project_files)) + .map(&method(:expand_variables)) .each(&method(:verify_duplicates!)) .map(&method(:select_first_matching)) end @@ -47,14 +48,14 @@ module Gitlab # convert location if String to canonical form def normalize_location(location) if location.is_a?(String) - normalize_location_string(location) + expanded_location = expand_variables(location) + normalize_location_string(expanded_location) else location.deep_symbolize_keys end end def expand_project_files(location) - return location unless ::Feature.enabled?(:ci_include_multiple_files_from_project, context.project, default_enabled: true) return location unless location[:project] Array.wrap(location[:file]).map do |file| @@ -96,6 +97,33 @@ module Gitlab matching.first end + + def expand_variables(data) + return data unless ::Feature.enabled?(:variables_in_include_section_ci) + + if data.is_a?(String) + expand(data) + else + transform(data) + end + end + + def transform(data) + data.transform_values do |values| + case values + when Array + values.map { |value| expand(value.to_s) } + when String + expand(values) + else + values + end + end + end + + def expand(data) + ExpandVariables.expand(data, context.variables) + end end end end diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index af1df933b36..7956cf14203 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -56,23 +56,19 @@ module Gitlab end def self.pipeline_open_merge_requests?(project) - ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: false) - end - - def self.seed_block_run_before_workflow_rules_enabled?(project) - ::Feature.enabled?(:ci_seed_block_run_before_workflow_rules, project, default_enabled: true) + ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: true) end def self.ci_pipeline_editor_page_enabled?(project) - ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: false) + ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: :yaml) end def self.allow_failure_with_exit_codes_enabled? - ::Feature.enabled?(:ci_allow_failure_with_exit_codes) + ::Feature.enabled?(:ci_allow_failure_with_exit_codes, default_enabled: :yaml) end def self.rules_variables_enabled?(project) - ::Feature.enabled?(:ci_rules_variables, project, default_enabled: false) + ::Feature.enabled?(:ci_rules_variables, project, default_enabled: true) end end end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index fb795152abe..364e67db02b 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -18,9 +18,10 @@ module Gitlab end end - def initialize(project:, current_user:) + def initialize(project:, current_user:, sha: nil) @project = project @current_user = current_user + @sha = sha || project.repository.commit.sha end def validate(content, dry_run: false) @@ -51,7 +52,7 @@ module Gitlab content, project: @project, user: @current_user, - sha: @project.repository.commit.sha + sha: @sha ).execute Result.new( @@ -99,7 +100,8 @@ module Gitlab except: job[:except], environment: job[:environment], when: job[:when], - allow_failure: job[:allow_failure] + allow_failure: job[:allow_failure], + needs: job.dig(:needs_attributes) } end end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 57f73c265b2..985639982aa 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -15,8 +15,8 @@ module Gitlab } end - def self.fabricate!(file_type) - parsers.fetch(file_type.to_sym).new + def self.fabricate!(file_type, *args) + parsers.fetch(file_type.to_sym).new(*args) rescue KeyError raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'" end diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb index 1edcbac2f25..eb3adf713d4 100644 --- a/lib/gitlab/ci/parsers/coverage/cobertura.rb +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -36,7 +36,7 @@ module Gitlab end def parse_node(key, value, coverage_report, context) - if key == 'sources' && value['source'].present? + if key == 'sources' && value && value['source'].present? parse_sources(value['source'], context) elsif key == 'package' Array.wrap(value).each do |item| diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb index 9662209f88e..f0548284001 100644 --- a/lib/gitlab/ci/pipeline/chain/build.rb +++ b/lib/gitlab/ci/pipeline/chain/build.rb @@ -5,6 +5,9 @@ module Gitlab module Pipeline module Chain class Build < Chain::Base + include Gitlab::Allowable + include Chain::Helpers + def perform! @pipeline.assign_attributes( source: @command.source, @@ -20,12 +23,34 @@ module Gitlab pipeline_schedule: @command.schedule, merge_request: @command.merge_request, external_pull_request: @command.external_pull_request, - variables_attributes: Array(@command.variables_attributes) + locked: @command.project.latest_pipeline_locked, + variables_attributes: variables_attributes ) end def break? - false + @pipeline.errors.any? + end + + private + + def variables_attributes + variables = Array(@command.variables_attributes) + + # We allow parent pipelines to pass variables to child pipelines since + # these variables are coming from internal configurations. We will check + # permissions to :set_pipeline_variables when those are injected upstream, + # to the parent pipeline. + # In other scenarios (e.g. multi-project pipelines or run pipeline via UI) + # the variables are provided from the outside and those should be guarded. + return variables if @command.creates_child_pipeline? + + if variables.present? && !can?(@command.current_user, :set_pipeline_variables, @command.project) + error("Insufficient permissions to set pipeline variables") + variables = [] + end + + variables end end end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index d05be54267c..815fe6bac6d 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -79,6 +79,10 @@ module Gitlab bridge&.parent_pipeline end + def creates_child_pipeline? + bridge&.triggers_child_pipeline? + end + def metrics @metrics ||= ::Gitlab::Ci::Pipeline::Metrics.new end diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb index 083f0bec1df..7b537125b9b 100644 --- a/lib/gitlab/ci/pipeline/chain/seed.rb +++ b/lib/gitlab/ci/pipeline/chain/seed.rb @@ -19,13 +19,6 @@ module Gitlab # Build to prevent erroring out on ambiguous refs. pipeline.protected = @command.protected_ref? - unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) - ## - # Populate pipeline with block argument of CreatePipelineService#execute. - # - @command.seeds_block&.call(pipeline) - end - ## # Gather all runtime build/stage errors # diff --git a/lib/gitlab/ci/pipeline/chain/seed_block.rb b/lib/gitlab/ci/pipeline/chain/seed_block.rb index f8e62949bea..67424635603 100644 --- a/lib/gitlab/ci/pipeline/chain/seed_block.rb +++ b/lib/gitlab/ci/pipeline/chain/seed_block.rb @@ -9,8 +9,6 @@ module Gitlab include Gitlab::Utils::StrongMemoize def perform! - return unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) - ## # Populate pipeline with block argument of CreatePipelineService#execute. # @@ -20,8 +18,6 @@ module Gitlab end def break? - return false unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project) - pipeline.errors.any? end end diff --git a/lib/gitlab/ci/pipeline/chain/template_usage.rb b/lib/gitlab/ci/pipeline/chain/template_usage.rb new file mode 100644 index 00000000000..c1a7b4ed453 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/template_usage.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class TemplateUsage < Chain::Base + def perform! + included_templates.each do |template| + track_event(template) + end + end + + def break? + false + end + + private + + def track_event(template) + Gitlab::UsageDataCounters::CiTemplateUniqueCounter + .track_unique_project_event(project_id: pipeline.project_id, template: template) + end + + def included_templates + command.yaml_processor_result.included_templates + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 8f1e690c081..e68d9020a21 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -19,7 +19,7 @@ module Gitlab end unless allowed_to_write_ref? - error("Insufficient permissions for protected ref '#{command.ref}'") + error("You do not have sufficient permission to run a pipeline on '#{command.ref}'. Please select a different branch or contact your administrator for assistance. <a href=https://docs.gitlab.com/ee/ci/pipelines/#pipeline-security-on-protected-branches>Learn more</a>".html_safe) end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 2271915a72b..fe3c2bca551 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -134,7 +134,7 @@ module Gitlab stage.seeds_names.include?(need[:name]) end - "#{name}: needs '#{need[:name]}'" unless result + "'#{name}' job needs '#{need[:name]}' job, but it was not added to the pipeline" unless result end.compact end diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb index beceac5423a..c024e794ad5 100644 --- a/lib/gitlab/ci/reports/test_failure_history.rb +++ b/lib/gitlab/ci/reports/test_failure_history.rb @@ -12,8 +12,6 @@ module Gitlab end def load! - return unless Feature.enabled?(:test_failure_history, project) - recent_failures_count.each do |key_hash, count| failed_test_cases[key_hash].set_recent_failures(count, project.default_branch_or_master) end diff --git a/lib/gitlab/ci/status/group/factory.rb b/lib/gitlab/ci/status/group/factory.rb index ee785856fdd..37e2b7320e2 100644 --- a/lib/gitlab/ci/status/group/factory.rb +++ b/lib/gitlab/ci/status/group/factory.rb @@ -8,6 +8,10 @@ module Gitlab def self.common_helpers Status::Group::Common end + + def self.extended_statuses + [[Status::SuccessWarning]] + end end end end diff --git a/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml new file mode 100644 index 00000000000..7182b96594d --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml @@ -0,0 +1,52 @@ +# +# You can use artifacts to pass data to jobs in later stages. +# For more information, see https://docs.gitlab.com/ee/ci/pipelines/job_artifacts.html +# + +stages: + - build + - test + - deploy + +build-job: + stage: build + script: + - echo "This job might build an important file, and pass it to later jobs." + - echo "This is the content of the important file" > important-file.txt + artifacts: + paths: + - important-file.txt + +test-job-with-artifacts: + stage: test + script: + - echo "This job uses the artifact from the job in the earlier stage." + - cat important-file.txt + - echo "It creates another file, and adds it to the artifacts." + - echo "This is a second important file" > important-file2.txt + artifacts: + paths: + - important-file2.txt + +test-job-with-no-artifacts: + stage: test + dependencies: [] # Use to skip downloading any artifacts + script: + - echo "This job does not get the artifacts from other jobs." + - cat important-file.txt || exit 0 + +deploy-job-with-all-artifacts: + stage: deploy + script: + - echo "By default, jobs download all available artifacts." + - cat important-file.txt + - cat important-file2.txt + +deploy-job-with-1-artifact: + stage: deploy + dependencies: + - build-job # Download artifacts from only this job + script: + - echo "You can configure a job to download artifacts from only certain jobs." + - cat important-file.txt + - cat important-file2.txt || exit 0 diff --git a/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml new file mode 100644 index 00000000000..382bac09ed7 --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml @@ -0,0 +1,36 @@ +# +# You can define common tasks and run them before or after the main scripts in jobs. +# For more information, see: +# - https://docs.gitlab.com/ee/ci/yaml/README.html#before_script +# - https://docs.gitlab.com/ee/ci/yaml/README.html#after_script +# + +stages: + - test + +default: + before_script: + - echo "This script runs before the main script in every job, unless the job overrides it." + - echo "It may set up common dependencies, for example." + after_script: + - echo "This script runs after the main script in every job, unless the job overrides it." + - echo "It may do some common final clean up tasks" + +job-standard: + stage: test + script: + - echo "This job uses both of the globally defined before and after scripts." + +job-override-before: + stage: test + before_script: + - echo "Use a different before_script in this job." + script: + - echo "This job uses its own before_script, and the global after_script." + +job-override-after: + stage: test + after_script: + - echo "Use a different after_script in this job." + script: + - echo "This job uses its own after_script, and the global before_script." diff --git a/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml new file mode 100644 index 00000000000..5f27def74c9 --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml @@ -0,0 +1,53 @@ +# +# A manual job is a type of job that is not executed automatically and must be explicitly started by a user. +# To make a job manual, add when: manual to its configuration. +# For more information, see https://docs.gitlab.com/ee/ci/yaml/README.html#whenmanual +# + +stages: + - build + - test + - deploy + +build-job: + stage: build + script: + - echo "This job is not a manual job" + +manual-build: + stage: build + script: + - echo "This manual job passes after you trigger it." + when: manual + +manual-build-allowed-to-fail: + stage: build + script: + - echo "This manual job fails after you trigger it." + - echo "It is allowed to fail, so the pipeline does not fail. + when: manual + allow_failure: true # Default behavior + +test-job: + stage: test + script: + - echo "This is a normal test job" + - echo "It runs when the when the build stage completes." + - echo "It does not need to wait for the manual jobs in the build stage to run." + +manual-test-not-allowed-to-fail: + stage: test + script: + - echo "This manual job fails after you trigger it." + - echo "It is NOT allowed to fail, so the pipeline is marked as failed + - echo "when this job completes." + - exit 1 + when: manual + allow_failure: false # Optional behavior + +deploy-job: + stage: deploy + script: + - echo "This is a normal deploy job" + - echo "If a manual job that isn't allowed to fail ran in an earlier stage and failed, + - echo "this job does not run". diff --git a/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml new file mode 100644 index 00000000000..aced628aacb --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml @@ -0,0 +1,33 @@ +# +# A pipeline is composed of independent jobs that run scripts, grouped into stages. +# Stages run in sequential order, but jobs within stages run in parallel. +# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages +# + +stages: + - build + - test + - deploy + +build-job: + stage: build + script: + - echo "This job runs in the build stage, which runs first." + +test-job1: + stage: test + script: + - echo "This job runs in the test stage." + - echo "It only starts when the job in the build stage completes successfully." + +test-job2: + stage: test + script: + - echo "This job also runs in the test stage." + - echo "This job can run at the same time as test-job2." + +deploy-job: + stage: deploy + script: + - echo "This job runs in the deploy stage." + - echo "It only runs when both jobs in the test stage complete successfully" diff --git a/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml new file mode 100644 index 00000000000..2b8cf7bab44 --- /dev/null +++ b/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml @@ -0,0 +1,47 @@ +# +# Variables can be used to for more dynamic behavior in jobs and scripts. +# For more information, see https://docs.gitlab.com/ee/ci/variables/README.html +# + +stages: + - test + +variables: + VAR1: "Variable 1 defined globally" + +use-a-variable: + stage: test + script: + - echo "You can use variables in jobs." + - echo "The content of 'VAR1' is = $VAR1" + +override-a-variable: + stage: test + variables: + VAR1: "Variable 1 was overriden in in the job." + script: + - echo "You can override global variables in jobs." + - echo "The content of 'VAR1' is = $VAR1" + +define-a-new-variable: + stage: test + variables: + VAR2: "Variable 2 is new and defined in the job only." + script: + - echo "You can mix global variables with variables defined in jobs." + - echo "The content of 'VAR1' is = $VAR1" + - echo "The content of 'VAR2' is = $VAR2" + +incorrect-variable-usage: + stage: test + script: + - echo "You can't use variables only defined in other jobs." + - echo "The content of 'VAR2' is = $VAR2" + +predefined-variables: + stage: test + script: + - echo "Some variables are predefined by GitLab CI/CD, for example:" + - echo "The commit author's username is $GITLAB_USER_LOGIN" + - echo "The commit branch is $CI_COMMIT_BRANCH" + - echo "The project path is $CI_PROJECT_PATH" diff --git a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml new file mode 100644 index 00000000000..c06ef83c180 --- /dev/null +++ b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml @@ -0,0 +1,84 @@ +# This template is on early stage of development. +# Use it with caution. For usage instruction please read +# https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v2.3.0/README.md + +include: + # workflow rules to prevent duplicate detached pipelines + - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml' + # auto devops build + - template: 'Jobs/Build.gitlab-ci.yml' + +stages: + - build + - test + - provision + - deploy + - destroy + +variables: + TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_COMMIT_REF_SLUG} + TF_VAR_ENVIRONMENT_NAME: ${CI_PROJECT_PATH_SLUG}_${CI_PROJECT_ID}_${CI_COMMIT_REF_SLUG} + TF_VAR_SERVICE_DESK_EMAIL: incoming+${CI_PROJECT_PATH_SLUG}-${CI_PROJECT_ID}-issue-@incoming.gitlab.com + TF_VAR_SHORT_ENVIRONMENT_NAME: ${CI_PROJECT_ID}-${CI_COMMIT_REF_SLUG} + TF_VAR_SMTP_FROM: ${SMTP_FROM} + +cache: + paths: + - .terraform + +.needs_aws_vars: + rules: + - if: '$AWS_ACCESS_KEY_ID && $AWS_SECRET_ACCESS_KEY && $AWS_DEFAULT_REGION' + when: on_success + - when: never + +terraform_apply: + stage: provision + image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + extends: .needs_aws_vars + resource_group: terraform + before_script: + - cp /*.tf . + - cp /deploy.sh . + script: + - gitlab-terraform init + - gitlab-terraform plan + - gitlab-terraform plan-json + - gitlab-terraform apply + +deploy: + stage: deploy + image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + extends: .needs_aws_vars + resource_group: deploy + before_script: + - cp /*.tf . + - cp /deploy.sh . + - cp /conf.nginx . + script: + - ./deploy.sh + artifacts: + reports: + dotenv: deploy.env + environment: + name: $CI_COMMIT_REF_SLUG + url: $DYNAMIC_ENVIRONMENT_URL + on_stop: terraform_destroy + +terraform_destroy: + variables: + GIT_STRATEGY: none + stage: destroy + image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable + before_script: + - cp /*.tf . + - cp /deploy.sh . + script: + - gitlab-terraform destroy -auto-approve + environment: + name: $CI_COMMIT_REF_SLUG + action: stop + rules: + - if: '$AWS_ACCESS_KEY_ID && $AWS_SECRET_ACCESS_KEY && $AWS_DEFAULT_REGION && $CI_COMMIT_REF_PROTECTED == "false"' + when: manual + - when: never diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml new file mode 100644 index 00000000000..504ece611ca --- /dev/null +++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml @@ -0,0 +1,29 @@ +code_quality: + stage: test + image: "cirrusci/flutter:1.22.5" + before_script: + - pub global activate dart_code_metrics + - export PATH="$PATH":"$HOME/.pub-cache/bin" + script: + - metrics lib -r codeclimate > gl-code-quality-report.json + artifacts: + reports: + codequality: gl-code-quality-report.json + +test: + stage: test + image: "cirrusci/flutter:1.22.5" + before_script: + - pub global activate junitreport + - export PATH="$PATH":"$HOME/.pub-cache/bin" + script: + - flutter test --machine --coverage | tojunit -o report.xml + - lcov --summary coverage/lcov.info + - genhtml coverage/lcov.info --output=coverage + coverage: '/lines\.*: \d+\.\d+\%/' + artifacts: + name: coverage + paths: + - $CI_PROJECT_DIR/coverage + reports: + junit: report.xml diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 2ae9730ec1a..501d8737acd 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -7,7 +7,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.18-gitlab.1" + CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.19" needs: [] script: - export SOURCE_CODE=$PWD diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml index 23dfeda31cc..192b1509fdc 100644 --- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml @@ -1,6 +1,6 @@ apply: stage: deploy - image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.36.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.37.0" environment: name: production variables: diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml new file mode 100644 index 00000000000..fc1acd09714 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -0,0 +1,43 @@ +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/ + +# Configure the scanning tool through the environment variables. +# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables +# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables + +variables: + DAST_VERSION: 1 + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" + +dast: + stage: dast + image: + name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION" + variables: + GIT_STRATEGY: none + allow_failure: true + script: + - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)} + - if [ -z "$DAST_WEBSITE$DAST_API_SPECIFICATION" ]; then echo "Either DAST_WEBSITE or DAST_API_SPECIFICATION must be set. See https://docs.gitlab.com/ee/user/application_security/dast/#configuration for more details." && exit 1; fi + - /analyze + artifacts: + reports: + dast: gl-dast-report.json + rules: + - if: $DAST_DISABLED + when: never + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && + $REVIEW_DISABLED && $DAST_WEBSITE == null && + $DAST_API_SPECIFICATION == null + when: never + - if: $CI_COMMIT_BRANCH && + $CI_KUBERNETES_ACTIVE && + $GITLAB_FEATURES =~ /\bdast\b/ + - if: $CI_COMMIT_BRANCH && + $DAST_WEBSITE + - if: $CI_COMMIT_BRANCH && + $DAST_API_SPECIFICATION diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml index f4ee8ebd47e..56c6fbd96bc 100644 --- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml @@ -10,6 +10,7 @@ variables: SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers" SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf" + SAST_EXCLUDED_ANALYZERS: "" SAST_EXCLUDED_PATHS: "spec, test, tests, tmp" SAST_ANALYZER_IMAGE_TAG: 2 SCAN_KUBERNETES_MANIFESTS: "false" @@ -44,6 +45,8 @@ bandit-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /bandit/ exists: @@ -58,6 +61,8 @@ brakeman-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /brakeman/ exists: @@ -72,6 +77,8 @@ eslint-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /eslint/ exists: @@ -90,6 +97,8 @@ flawfinder-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /flawfinder/ exists: @@ -105,6 +114,8 @@ kubesec-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /kubesec/ && $SCAN_KUBERNETES_MANIFESTS == 'true' @@ -118,6 +129,8 @@ gosec-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /gosec/ exists: @@ -136,6 +149,8 @@ mobsf-android-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' @@ -155,6 +170,8 @@ mobsf-ios-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' @@ -170,6 +187,8 @@ nodejs-scan-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/ exists: @@ -184,6 +203,8 @@ phpcs-security-audit-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/ exists: @@ -198,6 +219,8 @@ pmd-apex-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/ exists: @@ -212,6 +235,8 @@ security-code-scan-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/ exists: @@ -227,6 +252,8 @@ sobelow-sast: rules: - if: $SAST_DISABLED when: never + - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ + when: never - if: $CI_COMMIT_BRANCH && $SAST_DEFAULT_ANALYZERS =~ /sobelow/ exists: @@ -239,6 +266,8 @@ spotbugs-sast: variables: SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: + - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ + when: never - if: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ && $SAST_EXPERIMENTAL_FEATURES == 'true' exists: diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml index 8ca1d2e08ba..d2a6fa06dd8 100644 --- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml @@ -37,6 +37,7 @@ secret_detection: when: never - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH script: + - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 377c72e8031..7e2828d010f 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -17,6 +17,7 @@ variables: cache: paths: - .terraform + - .terraform.lock.hcl before_script: - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'" diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 910e711f046..c2db0fc44f1 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -19,6 +19,7 @@ cache: key: "${TF_ROOT}" paths: - ${TF_ROOT}/.terraform/ + - ${TF_ROOT}/.terraform.lock.hcl .init: &init stage: init diff --git a/lib/gitlab/ci/variables/collection/sorted.rb b/lib/gitlab/ci/variables/collection/sorted.rb new file mode 100644 index 00000000000..6abc6a5644f --- /dev/null +++ b/lib/gitlab/ci/variables/collection/sorted.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + class Collection + class Sorted + include TSort + include Gitlab::Utils::StrongMemoize + + def initialize(variables) + @variables = variables + end + + def valid? + errors.nil? + end + + # errors sorts an array of variables, ignoring unknown variable references, + # and returning an error string if a circular variable reference is found + def errors + return if Feature.disabled?(:variable_inside_variable) + + strong_memoize(:errors) do + # Check for cyclic dependencies and build error message in that case + errors = each_strongly_connected_component.filter_map do |component| + component.map { |v| v[:key] }.inspect if component.size > 1 + end + + "circular variable reference detected: #{errors.join(', ')}" if errors.any? + end + end + + # sort sorts an array of variables, ignoring unknown variable references. + # If a circular variable reference is found, the original array is returned + def sort + return @variables if Feature.disabled?(:variable_inside_variable) + return @variables if errors + + tsort + end + + private + + def tsort_each_node(&block) + @variables.each(&block) + end + + def tsort_each_child(variable, &block) + each_variable_reference(variable[:value], &block) + end + + def input_vars + strong_memoize(:input_vars) do + @variables.index_by { |env| env.fetch(:key) } + end + end + + def walk_references(value) + return unless ExpandVariables.possible_var_reference?(value) + + value.scan(ExpandVariables::VARIABLES_REGEXP) do |var_ref| + yield(input_vars, var_ref.first) + end + end + + def each_variable_reference(value) + walk_references(value) do |vars_hash, ref_var_name| + variable = vars_hash.dig(ref_var_name) + yield variable if variable + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index ee55eb8b22a..dc4951f76bb 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -10,12 +10,6 @@ module Gitlab class YamlProcessor ValidationError = Class.new(StandardError) - def self.validation_message(content, opts = {}) - result = new(content, opts).execute - - result.errors.first - end - def initialize(config_content, opts = {}) @config_content = config_content @opts = opts diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index cd7d781a574..86749cda9c7 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -53,6 +53,10 @@ module Gitlab @stages ||= @ci_config.stages end + def included_templates + @included_templates ||= @ci_config.included_templates + end + def build_attributes(name) job = jobs.fetch(name.to_sym, {}) diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb new file mode 100644 index 00000000000..de9a17a453f --- /dev/null +++ b/lib/gitlab/composer/version_index.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Composer + class VersionIndex + include API::Helpers::RelatedResourcesHelpers + + def initialize(packages) + @packages = packages + end + + def as_json(_options = nil) + { 'packages' => { @packages.first.name => package_versions_map } } + end + + def sha + Digest::SHA256.hexdigest(to_json) + end + + private + + def package_versions_map + @packages.each_with_object({}) do |package, map| + map[package.version] = package_metadata(package) + end + end + + def package_metadata(package) + json = package.composer_metadatum.composer_json + + json.merge('dist' => package_dist(package), 'uid' => package.id, 'version' => package.version) + end + + def package_dist(package) + sha = package.composer_metadatum.target_sha + archive_api_path = api_v4_projects_packages_composer_archives_package_name_path({ id: package.project_id, package_name: package.name, format: '.zip' }, true) + + { + 'type' => 'zip', + 'url' => expose_url(archive_api_path) + "?sha=#{sha}", + 'reference' => sha, + 'shasum' => '' + } + end + end + end +end diff --git a/lib/gitlab/conflict/file.rb b/lib/gitlab/conflict/file.rb index 4d7590a8e38..fbf021345ca 100644 --- a/lib/gitlab/conflict/file.rb +++ b/lib/gitlab/conflict/file.rb @@ -9,9 +9,13 @@ module Gitlab CONTEXT_LINES = 3 + CONFLICT_MARKER_OUR = 'conflict_marker_our' + CONFLICT_MARKER_THEIR = 'conflict_marker_their' + CONFLICT_MARKER_SEPARATOR = 'conflict_marker' + CONFLICT_TYPES = { - "old" => "conflict_marker_their", - "new" => "conflict_marker_our" + "old" => "conflict_their", + "new" => "conflict_our" }.freeze attr_reader :merge_request @@ -59,18 +63,25 @@ module Gitlab if section[:conflict] lines = [] - initial_type = nil + lines << create_separator_line(section[:lines].first, CONFLICT_MARKER_OUR) + + current_type = section[:lines].first.type section[:lines].each do |line| - if line.type != initial_type - lines << create_separator_line(line) - initial_type = line.type + if line.type != current_type # insert a separator between our changes and theirs + lines << create_separator_line(line, CONFLICT_MARKER_SEPARATOR) + current_type = line.type end line.type = CONFLICT_TYPES[line.type] + + # Swap the positions around due to conflicts/diffs display inconsistency + # https://gitlab.com/gitlab-org/gitlab/-/issues/291989 + line.old_pos, line.new_pos = line.new_pos, line.old_pos + lines << line end - lines << create_separator_line(lines.last) + lines << create_separator_line(lines.last, CONFLICT_MARKER_THEIR) lines else @@ -156,8 +167,8 @@ module Gitlab Gitlab::Diff::Line.new('', 'match', line.index, line.old_pos, line.new_pos) end - def create_separator_line(line) - Gitlab::Diff::Line.new('', 'conflict_marker', line.index, nil, nil) + def create_separator_line(line, type) + Gitlab::Diff::Line.new('', type, line.index, nil, nil) end # Any line beginning with a letter, an underscore, or a dollar can be used in a diff --git a/lib/gitlab/cycle_analytics/base_event_fetcher.rb b/lib/gitlab/cycle_analytics/base_event_fetcher.rb deleted file mode 100644 index 6c6dd90e450..00000000000 --- a/lib/gitlab/cycle_analytics/base_event_fetcher.rb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class BaseEventFetcher - include BaseQuery - include GroupProjectsProvider - - attr_reader :projections, :query, :stage, :options - - MAX_EVENTS = 50 - - def initialize(stage:, options:) - @stage = stage - @options = options - end - - def fetch - update_author! - - event_result.map do |event| - serialize(event) if has_permission?(event['id']) - end.compact - end - - def order - @order || default_order - end - - private - - def update_author! - return unless event_result.any? && event_result.first['author_id'] - - Updater.update!(event_result, from: 'author_id', to: 'author', klass: User) - end - - def event_result - @event_result ||= ActiveRecord::Base.connection.exec_query(events_query.to_sql).to_a - end - - def events_query - diff_fn = subtract_datetimes_diff(base_query, options[:start_time_attrs], options[:end_time_attrs]) - - base_query.project(extract_diff_epoch(diff_fn).as('total_time'), *projections).order(order.desc).take(MAX_EVENTS) - end - - def default_order - [options[:start_time_attrs]].flatten.first - end - - def serialize(_event) - raise NotImplementedError.new("Expected #{self.name} to implement serialize(event)") - end - - def has_permission?(id) - allowed_ids.nil? || allowed_ids.include?(id.to_i) - end - - def allowed_ids - @allowed_ids ||= allowed_ids_finder_class - .new(options[:current_user], allowed_ids_source) - .execute.where(id: event_result_ids).pluck(:id) - end - - def event_result_ids - event_result.map { |event| event['id'] } - end - - def allowed_ids_source - group ? { group_id: group.id, include_subgroups: true } : { project_id: project.id } - end - - def serialization_context - {} - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/base_query.rb b/lib/gitlab/cycle_analytics/base_query.rb deleted file mode 100644 index 6aedbf64f26..00000000000 --- a/lib/gitlab/cycle_analytics/base_query.rb +++ /dev/null @@ -1,54 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module BaseQuery - include MetricsTables - include Gitlab::Database::Median - include Gitlab::Database::DateTime - - private - - def base_query - @base_query ||= stage_query(projects.map(&:id)) - end - - def stage_query(project_ids) - query = mr_closing_issues_table.join(issue_table).on(issue_table[:id].eq(mr_closing_issues_table[:issue_id])) - .join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) - .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) - .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) - .project(issue_table[:project_id].as("project_id")) - .project(projects_table[:path].as("project_path")) - .project(routes_table[:path].as("namespace_path")) - - query = limit_query(query, project_ids) - query = limit_query_by_date_range(query) - - # Load merge_requests - - query = load_merge_requests(query) - - query - end - - def limit_query(query, project_ids) - query.where(issue_table[:project_id].in(project_ids)) - .where(routes_table[:source_type].eq('Namespace')) - end - - def limit_query_by_date_range(query) - query = query.where(issue_table[:created_at].gteq(options[:from])) - query = query.where(issue_table[:created_at].lteq(options[:to])) if options[:to] - query - end - - def load_merge_requests(query) - query.join(mr_table, Arel::Nodes::OuterJoin) - .on(mr_table[:id].eq(mr_closing_issues_table[:merge_request_id])) - .join(mr_metrics_table) - .on(mr_table[:id].eq(mr_metrics_table[:merge_request_id])) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/base_stage.rb b/lib/gitlab/cycle_analytics/base_stage.rb deleted file mode 100644 index 06f0cbed147..00000000000 --- a/lib/gitlab/cycle_analytics/base_stage.rb +++ /dev/null @@ -1,83 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class BaseStage - include BaseQuery - include GroupProjectsProvider - - attr_reader :options - - def initialize(options:) - @options = options - end - - def events - event_fetcher.fetch - end - - def as_json(serializer: AnalyticsStageSerializer) - serializer.new.represent(self) - end - - def title - raise NotImplementedError.new("Expected #{self.name} to implement title") - end - - def project_median - return if project.nil? - - BatchLoader.for(project.id).batch(key: name) do |project_ids, loader| - if project_ids.one? - loader.call(project.id, median_query(project_ids)) - else - begin - median_datetimes(cte_table, interval_query(project_ids), name, :project_id)&.each do |project_id, median| - loader.call(project_id, median) - end - rescue NotSupportedError - {} - end - end - end - end - - def group_median - median_query(projects.map(&:id)) - end - - def median_query(project_ids) - # Build a `SELECT` query. We find the first of the `end_time_attrs` that isn't `NULL` (call this end_time). - # Next, we find the first of the start_time_attrs that isn't `NULL` (call this start_time). - # We compute the (end_time - start_time) interval, and give it an alias based on the current - # value stream analytics stage. - - median_datetime(cte_table, interval_query(project_ids), name) - end - - def name - raise NotImplementedError.new("Expected #{self.name} to implement name") - end - - def cte_table - Arel::Table.new("cte_table_for_#{name}") - end - - def interval_query(project_ids) - Arel::Nodes::As.new(cte_table, - subtract_datetimes(stage_query(project_ids), start_time_attrs, end_time_attrs, name.to_s)) - end - - private - - def event_fetcher - @event_fetcher ||= Gitlab::CycleAnalytics::EventFetcher[name].new(stage: name, - options: event_options) - end - - def event_options - options.merge(start_time_attrs: start_time_attrs, end_time_attrs: end_time_attrs) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/builds_event_helper.rb b/lib/gitlab/cycle_analytics/builds_event_helper.rb deleted file mode 100644 index c39d41578e9..00000000000 --- a/lib/gitlab/cycle_analytics/builds_event_helper.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module BuildsEventHelper - def initialize(...) - @projections = [build_table[:id]] - @order = build_table[:created_at] - - super(...) - end - - def fetch - Updater.update!(event_result, from: 'id', to: 'build', klass: ::Ci::Build) - - super - end - - def events_query - base_query.join(build_table).on(mr_metrics_table[:pipeline_id].eq(build_table[:commit_id])) - - super - end - - private - - def allowed_ids - nil - end - - def serialize(event) - AnalyticsBuildSerializer.new.represent(event['build']) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/code_event_fetcher.rb b/lib/gitlab/cycle_analytics/code_event_fetcher.rb deleted file mode 100644 index 790bf32c6c7..00000000000 --- a/lib/gitlab/cycle_analytics/code_event_fetcher.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class CodeEventFetcher < BaseEventFetcher - include CodeHelper - - def initialize(...) - @projections = [mr_table[:title], - mr_table[:iid], - mr_table[:id], - mr_table[:created_at], - mr_table[:state_id], - mr_table[:author_id]] - @order = mr_table[:created_at] - - super(...) - end - - private - - def serialize(event) - AnalyticsMergeRequestSerializer.new(serialization_context).represent(event) - end - - def allowed_ids_finder_class - MergeRequestsFinder - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/code_helper.rb b/lib/gitlab/cycle_analytics/code_helper.rb deleted file mode 100644 index 8f28bdd2502..00000000000 --- a/lib/gitlab/cycle_analytics/code_helper.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module CodeHelper - def stage_query(project_ids) - super(project_ids).where(mr_table[:created_at].gteq(issue_metrics_table[:first_mentioned_in_commit_at])) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/code_stage.rb b/lib/gitlab/cycle_analytics/code_stage.rb deleted file mode 100644 index 89a6430221c..00000000000 --- a/lib/gitlab/cycle_analytics/code_stage.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class CodeStage < BaseStage - include CodeHelper - - def start_time_attrs - @start_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] - end - - def end_time_attrs - @end_time_attrs ||= mr_table[:created_at] - end - - def name - :code - end - - def title - s_('CycleAnalyticsStage|Code') - end - - def legend - _("Related Merge Requests") - end - - def description - _("Time until first merge request") - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/event_fetcher.rb b/lib/gitlab/cycle_analytics/event_fetcher.rb deleted file mode 100644 index 04f4b4f053f..00000000000 --- a/lib/gitlab/cycle_analytics/event_fetcher.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module EventFetcher - def self.[](stage_name) - CycleAnalytics.const_get("#{stage_name.to_s.camelize}EventFetcher", false) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb b/lib/gitlab/cycle_analytics/issue_event_fetcher.rb deleted file mode 100644 index fd04ec090b3..00000000000 --- a/lib/gitlab/cycle_analytics/issue_event_fetcher.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class IssueEventFetcher < BaseEventFetcher - include IssueHelper - - def initialize(...) - @projections = [issue_table[:title], - issue_table[:iid], - issue_table[:id], - issue_table[:created_at], - issue_table[:author_id]] - - super(...) - end - - private - - def serialize(event) - AnalyticsIssueSerializer.new(serialization_context).represent(event) - end - - def allowed_ids_finder_class - IssuesFinder - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/issue_helper.rb b/lib/gitlab/cycle_analytics/issue_helper.rb deleted file mode 100644 index f6f85b84ed8..00000000000 --- a/lib/gitlab/cycle_analytics/issue_helper.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module IssueHelper - def stage_query(project_ids) - query = issue_table.join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) - .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) - .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) - .project(issue_table[:project_id].as("project_id")) - .project(projects_table[:path].as("project_path")) - .project(routes_table[:path].as("namespace_path")) - - query = limit_query(query, project_ids) - limit_query_by_date_range(query) - end - - def limit_query(query, project_ids) - query.where(issue_table[:project_id].in(project_ids)) - .where(routes_table[:source_type].eq('Namespace')) - .where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/issue_stage.rb b/lib/gitlab/cycle_analytics/issue_stage.rb deleted file mode 100644 index 738cb3eba03..00000000000 --- a/lib/gitlab/cycle_analytics/issue_stage.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class IssueStage < BaseStage - include IssueHelper - - def start_time_attrs - @start_time_attrs ||= issue_table[:created_at] - end - - def end_time_attrs - @end_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], - issue_metrics_table[:first_added_to_board_at]] - end - - def name - :issue - end - - def title - s_('CycleAnalyticsStage|Issue') - end - - def legend - _("Related Issues") - end - - def description - _("Time before an issue gets scheduled") - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/permissions.rb b/lib/gitlab/cycle_analytics/permissions.rb index 0e094fabb01..9164c8b1bff 100644 --- a/lib/gitlab/cycle_analytics/permissions.rb +++ b/lib/gitlab/cycle_analytics/permissions.rb @@ -23,7 +23,7 @@ module Gitlab end def get - ::CycleAnalytics::LevelBase::STAGES.each do |stage| + Gitlab::Analytics::CycleAnalytics::DefaultStages.symbolized_stage_names.each do |stage| @stage_permission_hash[stage] = authorized_stage?(stage) end diff --git a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb b/lib/gitlab/cycle_analytics/plan_event_fetcher.rb deleted file mode 100644 index 4d98d589e46..00000000000 --- a/lib/gitlab/cycle_analytics/plan_event_fetcher.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class PlanEventFetcher < BaseEventFetcher - include PlanHelper - - def initialize(...) - @projections = [issue_table[:title], - issue_table[:iid], - issue_table[:id], - issue_table[:created_at], - issue_table[:author_id]] - - super(...) - end - - private - - def serialize(event) - AnalyticsIssueSerializer.new(serialization_context).represent(event) - end - - def allowed_ids_finder_class - IssuesFinder - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/plan_helper.rb b/lib/gitlab/cycle_analytics/plan_helper.rb deleted file mode 100644 index af4bf6ed3eb..00000000000 --- a/lib/gitlab/cycle_analytics/plan_helper.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module PlanHelper - def stage_query(project_ids) - query = issue_table.join(issue_metrics_table).on(issue_table[:id].eq(issue_metrics_table[:issue_id])) - .join(projects_table).on(issue_table[:project_id].eq(projects_table[:id])) - .join(routes_table).on(projects_table[:namespace_id].eq(routes_table[:source_id])) - .project(issue_table[:project_id].as("project_id")) - .project(projects_table[:path].as("project_path")) - .project(routes_table[:path].as("namespace_path")) - .where(issue_table[:project_id].in(project_ids)) - .where(routes_table[:source_type].eq('Namespace')) - query = limit_query(query) - - limit_query_by_date_range(query) - end - - def limit_query(query) - query.where(issue_metrics_table[:first_added_to_board_at].not_eq(nil).or(issue_metrics_table[:first_associated_with_milestone_at].not_eq(nil))) - .where(issue_metrics_table[:first_mentioned_in_commit_at].not_eq(nil)) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/plan_stage.rb b/lib/gitlab/cycle_analytics/plan_stage.rb deleted file mode 100644 index 0b27d114f52..00000000000 --- a/lib/gitlab/cycle_analytics/plan_stage.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class PlanStage < BaseStage - include PlanHelper - - def start_time_attrs - @start_time_attrs ||= [issue_metrics_table[:first_associated_with_milestone_at], - issue_metrics_table[:first_added_to_board_at]] - end - - def end_time_attrs - @end_time_attrs ||= issue_metrics_table[:first_mentioned_in_commit_at] - end - - def name - :plan - end - - def title - s_('CycleAnalyticsStage|Plan') - end - - def legend - _("Related Issues") - end - - def description - _("Time before an issue starts implementation") - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/production_event_fetcher.rb b/lib/gitlab/cycle_analytics/production_event_fetcher.rb deleted file mode 100644 index 5fa286bd3df..00000000000 --- a/lib/gitlab/cycle_analytics/production_event_fetcher.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class ProductionEventFetcher < BaseEventFetcher - include ProductionHelper - - def initialize(...) - @projections = [issue_table[:title], - issue_table[:iid], - issue_table[:id], - issue_table[:created_at], - issue_table[:author_id], - routes_table[:path]] - - super(...) - end - - private - - def serialize(event) - AnalyticsIssueSerializer.new(serialization_context).represent(event) - end - - def allowed_ids_finder_class - IssuesFinder - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/production_helper.rb b/lib/gitlab/cycle_analytics/production_helper.rb deleted file mode 100644 index 778757a9ede..00000000000 --- a/lib/gitlab/cycle_analytics/production_helper.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module ProductionHelper - def stage_query(project_ids) - super(project_ids) - .where(mr_metrics_table[:first_deployed_to_production_at] - .gteq(options[:from])) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/review_event_fetcher.rb b/lib/gitlab/cycle_analytics/review_event_fetcher.rb deleted file mode 100644 index 0b7d160c7de..00000000000 --- a/lib/gitlab/cycle_analytics/review_event_fetcher.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class ReviewEventFetcher < BaseEventFetcher - include ReviewHelper - - def initialize(...) - @projections = [mr_table[:title], - mr_table[:iid], - mr_table[:id], - mr_table[:created_at], - mr_table[:state_id], - mr_table[:author_id]] - - super(...) - end - - private - - def serialize(event) - AnalyticsMergeRequestSerializer.new(serialization_context).represent(event) - end - - def allowed_ids_finder_class - MergeRequestsFinder - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/review_helper.rb b/lib/gitlab/cycle_analytics/review_helper.rb deleted file mode 100644 index c53249652b5..00000000000 --- a/lib/gitlab/cycle_analytics/review_helper.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module ReviewHelper - def stage_query(project_ids) - super(project_ids).where(mr_metrics_table[:merged_at].not_eq(nil)) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/review_stage.rb b/lib/gitlab/cycle_analytics/review_stage.rb deleted file mode 100644 index e9df8cd5a05..00000000000 --- a/lib/gitlab/cycle_analytics/review_stage.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class ReviewStage < BaseStage - include ReviewHelper - - def start_time_attrs - @start_time_attrs ||= mr_table[:created_at] - end - - def end_time_attrs - @end_time_attrs ||= mr_metrics_table[:merged_at] - end - - def name - :review - end - - def title - s_('CycleAnalyticsStage|Review') - end - - def legend - _("Related Merged Requests") - end - - def description - _("Time between merge request creation and merge/close") - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/stage.rb b/lib/gitlab/cycle_analytics/stage.rb deleted file mode 100644 index 5cfd9ea4730..00000000000 --- a/lib/gitlab/cycle_analytics/stage.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module Stage - def self.[](stage_name) - CycleAnalytics.const_get("#{stage_name.to_s.camelize}Stage", false) - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb b/lib/gitlab/cycle_analytics/staging_event_fetcher.rb deleted file mode 100644 index 1454a1a33eb..00000000000 --- a/lib/gitlab/cycle_analytics/staging_event_fetcher.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class StagingEventFetcher < BaseEventFetcher - include ProductionHelper - include BuildsEventHelper - end - end -end diff --git a/lib/gitlab/cycle_analytics/staging_stage.rb b/lib/gitlab/cycle_analytics/staging_stage.rb deleted file mode 100644 index e03627c6cd1..00000000000 --- a/lib/gitlab/cycle_analytics/staging_stage.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class StagingStage < BaseStage - include ProductionHelper - - def start_time_attrs - @start_time_attrs ||= mr_metrics_table[:merged_at] - end - - def end_time_attrs - @end_time_attrs ||= mr_metrics_table[:first_deployed_to_production_at] - end - - def name - :staging - end - - def title - s_('CycleAnalyticsStage|Staging') - end - - def legend - _("Related Deployed Jobs") - end - - def description - _("From merge request merge until deploy to production") - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/test_event_fetcher.rb b/lib/gitlab/cycle_analytics/test_event_fetcher.rb deleted file mode 100644 index 2fa44b1b364..00000000000 --- a/lib/gitlab/cycle_analytics/test_event_fetcher.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class TestEventFetcher < BaseEventFetcher - include TestHelper - include BuildsEventHelper - end - end -end diff --git a/lib/gitlab/cycle_analytics/test_helper.rb b/lib/gitlab/cycle_analytics/test_helper.rb deleted file mode 100644 index d9124d62c7c..00000000000 --- a/lib/gitlab/cycle_analytics/test_helper.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - module TestHelper - def stage_query(project_ids) - if branch - super(project_ids).where(build_table[:ref].eq(branch)) - else - super(project_ids) - end - end - - private - - def branch - @branch ||= options[:branch] - end - end - end -end diff --git a/lib/gitlab/cycle_analytics/test_stage.rb b/lib/gitlab/cycle_analytics/test_stage.rb deleted file mode 100644 index 4787a906c07..00000000000 --- a/lib/gitlab/cycle_analytics/test_stage.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module CycleAnalytics - class TestStage < BaseStage - include TestHelper - - def start_time_attrs - @start_time_attrs ||= mr_metrics_table[:latest_build_started_at] - end - - def end_time_attrs - @end_time_attrs ||= mr_metrics_table[:latest_build_finished_at] - end - - def name - :test - end - - def title - s_('CycleAnalyticsStage|Test') - end - - def legend - _("Related Jobs") - end - - def description - _("Total test time for all commits/merges") - end - end - end -end diff --git a/lib/gitlab/danger/base_linter.rb b/lib/gitlab/danger/base_linter.rb index df2e9e745aa..898434724bd 100644 --- a/lib/gitlab/danger/base_linter.rb +++ b/lib/gitlab/danger/base_linter.rb @@ -1,11 +1,12 @@ # frozen_string_literal: true +require_relative 'title_linting' + module Gitlab module Danger class BaseLinter MIN_SUBJECT_WORDS_COUNT = 3 MAX_LINE_LENGTH = 72 - WIP_PREFIX = 'WIP: ' attr_reader :commit, :problems @@ -58,7 +59,7 @@ module Gitlab private def subject - message_parts[0].delete_prefix(WIP_PREFIX) + TitleLinting.remove_draft_flag(message_parts[0]) end def subject_too_short? diff --git a/lib/gitlab/danger/changelog.rb b/lib/gitlab/danger/changelog.rb index 92af6849b2f..4b85775ed98 100644 --- a/lib/gitlab/danger/changelog.rb +++ b/lib/gitlab/danger/changelog.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative 'title_linting' + module Gitlab module Danger module Changelog @@ -75,7 +77,7 @@ module Gitlab end def sanitized_mr_title - helper.sanitize_mr_title(gitlab.mr_json["title"]) + TitleLinting.sanitize_mr_title(gitlab.mr_json["title"]) end def categories_need_changelog? diff --git a/lib/gitlab/danger/commit_linter.rb b/lib/gitlab/danger/commit_linter.rb index 7e2e0fb0acb..e23f5900433 100644 --- a/lib/gitlab/danger/commit_linter.rb +++ b/lib/gitlab/danger/commit_linter.rb @@ -1,9 +1,15 @@ # frozen_string_literal: true -require_relative 'base_linter' - emoji_checker_path = File.expand_path('emoji_checker', __dir__) -defined?(Rails) ? require_dependency(emoji_checker_path) : require_relative(emoji_checker_path) +base_linter_path = File.expand_path('base_linter', __dir__) + +if defined?(Rails) + require_dependency(base_linter_path) + require_dependency(emoji_checker_path) +else + require_relative(base_linter_path) + require_relative(emoji_checker_path) +end module Gitlab module Danger diff --git a/lib/gitlab/danger/helper.rb b/lib/gitlab/danger/helper.rb index d22f28ff7f2..09e013e24b8 100644 --- a/lib/gitlab/danger/helper.rb +++ b/lib/gitlab/danger/helper.rb @@ -1,12 +1,12 @@ # frozen_string_literal: true require_relative 'teammate' +require_relative 'title_linting' module Gitlab module Danger module Helper RELEASE_TOOLS_BOT = 'gitlab-release-tools-bot' - DRAFT_REGEX = /\A*#{Regexp.union(/(?i)(\[WIP\]\s*|WIP:\s*|WIP$)/, /(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft$)/)}+\s*/i.freeze # Returns a list of all files that have been added, modified or renamed. # `git.modified_files` might contain paths that already have been renamed, @@ -128,7 +128,7 @@ module Gitlab }.freeze # First-match win, so be sure to put more specific regex at the top... CATEGORIES = { - [%r{usage_data\.rb}, %r{^(\+|-).*(count|distinct_count)\(.*\)(.*)$}] => [:database, :backend], + [%r{usage_data\.rb}, %r{^(\+|-).*\s+(count|distinct_count|estimate_batch_distinct_count)\(.*\)(.*)$}] => [:database, :backend], %r{\Adoc/.*(\.(md|png|gif|jpg))\z} => :docs, %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, @@ -216,14 +216,10 @@ module Gitlab usernames.map { |u| Gitlab::Danger::Teammate.new('username' => u) } end - def sanitize_mr_title(title) - title.gsub(DRAFT_REGEX, '').gsub(/`/, '\\\`') - end - def draft_mr? return false unless gitlab_helper - DRAFT_REGEX.match?(gitlab_helper.mr_json['title']) + TitleLinting.has_draft_flag?(gitlab_helper.mr_json['title']) end def security_mr? diff --git a/lib/gitlab/danger/merge_request_linter.rb b/lib/gitlab/danger/merge_request_linter.rb index d401d332aa7..ed354bfc68d 100644 --- a/lib/gitlab/danger/merge_request_linter.rb +++ b/lib/gitlab/danger/merge_request_linter.rb @@ -1,6 +1,12 @@ # frozen_string_literal: true -require_relative 'base_linter' +base_linter_path = File.expand_path('base_linter', __dir__) + +if defined?(Rails) + require_dependency(base_linter_path) +else + require_relative(base_linter_path) +end module Gitlab module Danger diff --git a/lib/gitlab/danger/roulette.rb b/lib/gitlab/danger/roulette.rb index 328083f7002..21feda2cf20 100644 --- a/lib/gitlab/danger/roulette.rb +++ b/lib/gitlab/danger/roulette.rb @@ -2,6 +2,8 @@ require_relative 'teammate' require_relative 'request_helper' unless defined?(Gitlab::Danger::RequestHelper) +require_relative 'weightage/reviewers' +require_relative 'weightage/maintainers' module Gitlab module Danger @@ -151,20 +153,14 @@ module Gitlab %i[reviewer traintainer maintainer].map do |role| spin_role_for_category(team, role, project, category) end - hungry_reviewers = reviewers.select { |member| member.hungry } - hungry_traintainers = traintainers.select { |member| member.hungry } - - # TODO: take CODEOWNERS into account? - # https://gitlab.com/gitlab-org/gitlab/issues/26723 random = new_random(mr_source_branch) - # Make hungry traintainers have 4x the chance to be picked as a reviewer - # Make traintainers have 3x the chance to be picked as a reviewer - # Make hungry reviewers have 2x the chance to be picked as a reviewer - weighted_reviewers = reviewers + hungry_reviewers + traintainers + traintainers + traintainers + hungry_traintainers + weighted_reviewers = Weightage::Reviewers.new(reviewers, traintainers).execute + weighted_maintainers = Weightage::Maintainers.new(maintainers).execute + reviewer = spin_for_person(weighted_reviewers, random: random, timezone_experiment: timezone_experiment) - maintainer = spin_for_person(maintainers, random: random, timezone_experiment: timezone_experiment) + maintainer = spin_for_person(weighted_maintainers, random: random, timezone_experiment: timezone_experiment) Spin.new(category, reviewer, maintainer, false, timezone_experiment) end diff --git a/lib/gitlab/danger/teammate.rb b/lib/gitlab/danger/teammate.rb index 4481977db15..911b84d93ec 100644 --- a/lib/gitlab/danger/teammate.rb +++ b/lib/gitlab/danger/teammate.rb @@ -3,7 +3,7 @@ module Gitlab module Danger class Teammate - attr_reader :options, :username, :name, :role, :projects, :available, :hungry, :tz_offset_hours + attr_reader :options, :username, :name, :role, :projects, :available, :hungry, :reduced_capacity, :tz_offset_hours # The options data are produced by https://gitlab.com/gitlab-org/gitlab-roulette/-/blob/master/lib/team_member.rb def initialize(options = {}) @@ -15,6 +15,7 @@ module Gitlab @projects = options['projects'] @available = options['available'] @hungry = options['hungry'] + @reduced_capacity = options['reduced_capacity'] @tz_offset_hours = options['tz_offset_hours'] end @@ -94,6 +95,7 @@ module Gitlab when :engineering_productivity return false unless role[/Engineering Productivity/] return true if kind == :reviewer + return true if capabilities(project).include?("#{kind} engineering_productivity") capabilities(project).include?("#{kind} backend") else diff --git a/lib/gitlab/danger/title_linting.rb b/lib/gitlab/danger/title_linting.rb new file mode 100644 index 00000000000..db1ccaaf9a9 --- /dev/null +++ b/lib/gitlab/danger/title_linting.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Danger + module TitleLinting + DRAFT_REGEX = /\A*#{Regexp.union(/(?i)(\[WIP\]\s*|WIP:\s*|WIP$)/, /(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft$)/)}+\s*/i.freeze + + module_function + + def sanitize_mr_title(title) + remove_draft_flag(title).gsub(/`/, '\\\`') + end + + def remove_draft_flag(title) + title.gsub(DRAFT_REGEX, '') + end + + def has_draft_flag?(title) + DRAFT_REGEX.match?(title) + end + end + end +end diff --git a/lib/gitlab/danger/weightage.rb b/lib/gitlab/danger/weightage.rb new file mode 100644 index 00000000000..67fade27573 --- /dev/null +++ b/lib/gitlab/danger/weightage.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module Danger + module Weightage + CAPACITY_MULTIPLIER = 2 # change this number to change what it means to be a reduced capacity reviewer 1/this number + BASE_REVIEWER_WEIGHT = 1 + end + end +end diff --git a/lib/gitlab/danger/weightage/maintainers.rb b/lib/gitlab/danger/weightage/maintainers.rb new file mode 100644 index 00000000000..cc0eb370e7a --- /dev/null +++ b/lib/gitlab/danger/weightage/maintainers.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require_relative '../weightage' + +module Gitlab + module Danger + module Weightage + class Maintainers + def initialize(maintainers) + @maintainers = maintainers + end + + def execute + maintainers.each_with_object([]) do |maintainer, weighted_maintainers| + add_weighted_reviewer(weighted_maintainers, maintainer, BASE_REVIEWER_WEIGHT) + end + end + + private + + attr_reader :maintainers + + def add_weighted_reviewer(reviewers, reviewer, weight) + if reviewer.reduced_capacity + reviewers.fill(reviewer, reviewers.size, weight) + else + reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER) + end + end + end + end + end +end diff --git a/lib/gitlab/danger/weightage/reviewers.rb b/lib/gitlab/danger/weightage/reviewers.rb new file mode 100644 index 00000000000..c8019be716e --- /dev/null +++ b/lib/gitlab/danger/weightage/reviewers.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require_relative '../weightage' + +module Gitlab + module Danger + module Weightage + # Weights after (current multiplier of 2) + # + # +------------------------------+--------------------------------+ + # | reviewer type | weight(times in reviewer pool) | + # +------------------------------+--------------------------------+ + # | reduced capacity reviewer | 1 | + # | reviewer | 2 | + # | hungry reviewer | 4 | + # | reduced capacity traintainer | 3 | + # | traintainer | 6 | + # | hungry traintainer | 8 | + # +------------------------------+--------------------------------+ + # + class Reviewers + DEFAULT_REVIEWER_WEIGHT = CAPACITY_MULTIPLIER * BASE_REVIEWER_WEIGHT + TRAINTAINER_WEIGHT = 3 + + def initialize(reviewers, traintainers) + @reviewers = reviewers + @traintainers = traintainers + end + + def execute + # TODO: take CODEOWNERS into account? + # https://gitlab.com/gitlab-org/gitlab/issues/26723 + + weighted_reviewers + weighted_traintainers + end + + private + + attr_reader :reviewers, :traintainers + + def weighted_reviewers + reviewers.each_with_object([]) do |reviewer, total_reviewers| + add_weighted_reviewer(total_reviewers, reviewer, BASE_REVIEWER_WEIGHT) + end + end + + def weighted_traintainers + traintainers.each_with_object([]) do |reviewer, total_traintainers| + add_weighted_reviewer(total_traintainers, reviewer, TRAINTAINER_WEIGHT) + end + end + + def add_weighted_reviewer(reviewers, reviewer, weight) + if reviewer.reduced_capacity + reviewers.fill(reviewer, reviewers.size, weight) + elsif reviewer.hungry + reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER + DEFAULT_REVIEWER_WEIGHT) + else + reviewers.fill(reviewer, reviewers.size, weight * CAPACITY_MULTIPLIER) + end + end + end + end + end +end diff --git a/lib/gitlab/database/median.rb b/lib/gitlab/database/median.rb deleted file mode 100644 index 603b125d8b4..00000000000 --- a/lib/gitlab/database/median.rb +++ /dev/null @@ -1,149 +0,0 @@ -# frozen_string_literal: true - -# https://www.periscopedata.com/blog/medians-in-sql.html -module Gitlab - module Database - module Median - NotSupportedError = Class.new(StandardError) - - def median_datetime(arel_table, query_so_far, column_sym) - extract_median(execute_queries(arel_table, query_so_far, column_sym)).presence - end - - def median_datetimes(arel_table, query_so_far, column_sym, partition_column) - extract_medians(execute_queries(arel_table, query_so_far, column_sym, partition_column)).presence - end - - def extract_median(results) - result = results.compact.first - - result = result.first.presence - - result['median']&.to_f if result - end - - def extract_medians(results) - median_values = results.compact.first.values - - median_values.each_with_object({}) do |(id, median), hash| - hash[id.to_i] = median&.to_f - end - end - - def pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column = nil) - # Create a CTE with the column we're operating on, row number (after sorting by the column - # we're operating on), and count of the table we're operating on (duplicated across) all rows - # of the CTE. For example, if we're looking to find the median of the `projects.star_count` - # column, the CTE might look like this: - # - # star_count | row_id | ct - # ------------+--------+---- - # 5 | 1 | 3 - # 9 | 2 | 3 - # 15 | 3 | 3 - # - # If a partition column is used we will do the same operation but for separate partitions, - # when that happens the CTE might look like this: - # - # project_id | star_count | row_id | ct - # ------------+------------+--------+---- - # 1 | 5 | 1 | 2 - # 1 | 9 | 2 | 2 - # 2 | 10 | 1 | 3 - # 2 | 15 | 2 | 3 - # 2 | 20 | 3 | 3 - cte_table = Arel::Table.new("ordered_records") - - cte = Arel::Nodes::As.new( - cte_table, - arel_table.project(*rank_rows(arel_table, column_sym, partition_column)). - # Disallow negative values - where(arel_table[column_sym].gteq(zero_interval))) - - # From the CTE, select either the middle row or the middle two rows (this is accomplished - # by 'where cte.row_id between cte.ct / 2.0 AND cte.ct / 2.0 + 1'). Find the average of the - # selected rows, and this is the median value. - result = - cte_table - .project(*median_projections(cte_table, column_sym, partition_column)) - .where( - Arel::Nodes::Between.new( - cte_table[:row_id], - Arel::Nodes::And.new( - [(cte_table[:ct] / Arel.sql('2.0')), - (cte_table[:ct] / Arel.sql('2.0') + 1)] - ) - ) - ) - .with(query_so_far, cte) - - result.group(cte_table[partition_column]).order(cte_table[partition_column]) if partition_column - - result.to_sql - end - - private - - def execute_queries(arel_table, query_so_far, column_sym, partition_column = nil) - queries = pg_median_datetime_sql(arel_table, query_so_far, column_sym, partition_column) - - Array.wrap(queries).map { |query| ActiveRecord::Base.connection.execute(query) } - end - - def average(args, as) - Arel::Nodes::NamedFunction.new("AVG", args, as) - end - - def rank_rows(arel_table, column_sym, partition_column) - column_row = arel_table[column_sym].as(column_sym.to_s) - - if partition_column - partition_row = arel_table[partition_column] - row_id = - Arel::Nodes::Over.new( - Arel::Nodes::NamedFunction.new('rank', []), - Arel::Nodes::Window.new.partition(arel_table[partition_column]) - .order(arel_table[column_sym]) - ).as('row_id') - - count = arel_table.from.from(arel_table.alias) - .project('COUNT(*)') - .where(arel_table[partition_column].eq(arel_table.alias[partition_column])) - .as('ct') - - [partition_row, column_row, row_id, count] - else - row_id = - Arel::Nodes::Over.new( - Arel::Nodes::NamedFunction.new('row_number', []), - Arel::Nodes::Window.new.order(arel_table[column_sym]) - ).as('row_id') - - count = arel_table.where(arel_table[column_sym].gteq(zero_interval)).project("COUNT(1)").as('ct') - - [column_row, row_id, count] - end - end - - def median_projections(table, column_sym, partition_column) - projections = [] - projections << table[partition_column] if partition_column - projections << average([extract_epoch(table[column_sym])], "median") - projections - end - - def extract_epoch(arel_attribute) - Arel.sql(%Q{EXTRACT(EPOCH FROM "#{arel_attribute.relation.name}"."#{arel_attribute.name}")}) - end - - def extract_diff_epoch(diff) - Arel.sql(%Q{EXTRACT(EPOCH FROM (#{diff.to_sql}))}) - end - - # Need to cast '0' to an INTERVAL before we can check if the interval is positive - def zero_interval - Arel::Nodes::NamedFunction.new("CAST", [Arel.sql("'0' AS INTERVAL")]) - end - end - end -end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 164fce5a5a3..6b169a504f3 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -70,6 +70,61 @@ module Gitlab end end + # + # Creates a new table, optionally allowing the caller to add check constraints to the table. + # Aside from that addition, this method should behave identically to Rails' `create_table` method. + # + # Example: + # + # create_table_with_constraints :some_table do |t| + # t.integer :thing, null: false + # t.text :other_thing + # + # t.check_constraint :thing_is_not_null, 'thing IS NOT NULL' + # t.text_limit :other_thing, 255 + # end + # + # See Rails' `create_table` for more info on the available arguments. + def create_table_with_constraints(table_name, **options, &block) + helper_context = self + check_constraints = [] + + with_lock_retries do + create_table(table_name, **options) do |t| + t.define_singleton_method(:check_constraint) do |name, definition| + helper_context.send(:validate_check_constraint_name!, name) # rubocop:disable GitlabSecurity/PublicSend + + check_constraints << { name: name, definition: definition } + end + + t.define_singleton_method(:text_limit) do |column_name, limit, name: nil| + # rubocop:disable GitlabSecurity/PublicSend + name = helper_context.send(:text_limit_name, table_name, column_name, name: name) + helper_context.send(:validate_check_constraint_name!, name) + # rubocop:enable GitlabSecurity/PublicSend + + column_name = helper_context.quote_column_name(column_name) + definition = "char_length(#{column_name}) <= #{limit}" + + check_constraints << { name: name, definition: definition } + end + + t.instance_eval(&block) unless block.nil? + end + + next if check_constraints.empty? + + constraint_clauses = check_constraints.map do |constraint| + "ADD CONSTRAINT #{quote_table_name(constraint[:name])} CHECK (#{constraint[:definition]})" + end + + execute(<<~SQL) + ALTER TABLE #{quote_table_name(table_name)} + #{constraint_clauses.join(",\n")} + SQL + end + end + # Creates a new index, concurrently # # Example: @@ -858,6 +913,120 @@ module Gitlab end end + # Initializes the conversion of an integer column to bigint + # + # It can be used for converting both a Primary Key and any Foreign Keys + # that may reference it or any other integer column that we may want to + # upgrade (e.g. columns that store IDs, but are not set as FKs). + # + # - For primary keys and Foreign Keys (or other columns) defined as NOT NULL, + # the new bigint column is added with a hardcoded NOT NULL DEFAULT 0 + # which allows us to skip a very costly verification step once we + # are ready to switch it. + # This is crucial for Primary Key conversions, because setting a column + # as the PK converts even check constraints to NOT NULL constraints + # and forces an inline re-verification of the whole table. + # - It backfills the new column with the values of the existing primary key + # by scheduling background jobs. + # - It tracks the scheduled background jobs through the use of + # Gitlab::Database::BackgroundMigrationJob + # which allows a more thorough check that all jobs succeeded in the + # cleanup migration and is way faster for very large tables. + # - It sets up a trigger to keep the two columns in sync + # - It does not schedule a cleanup job: we have to do that with followup + # post deployment migrations in the next release. + # + # This needs to be done manually by using the + # `cleanup_initialize_conversion_of_integer_to_bigint` + # (not yet implemented - check #288005) + # + # table - The name of the database table containing the column + # column - The name of the column that we want to convert to bigint. + # primary_key - The name of the primary key column (most often :id) + # batch_size - The number of rows to schedule in a single background migration + # sub_batch_size - The smaller batches that will be used by each scheduled job + # to update the table. Useful to keep each update at ~100ms while executing + # more updates per interval (2.minutes) + # Note that each execution of a sub-batch adds a constant 100ms sleep + # time in between the updates, which must be taken into account + # while calculating the batch, sub_batch and interval values. + # interval - The time interval between every background migration + # + # example: + # Assume that we have figured out that updating 200 records of the events + # table takes ~100ms on average. + # We can set the sub_batch_size to 200, leave the interval to the default + # and set the batch_size to 50_000 which will require + # ~50s = (50000 / 200) * (0.1 + 0.1) to complete and leaves breathing space + # between the scheduled jobs + def initialize_conversion_of_integer_to_bigint( + table, + column, + primary_key: :id, + batch_size: 20_000, + sub_batch_size: 1000, + interval: 2.minutes + ) + + if transaction_open? + raise 'initialize_conversion_of_integer_to_bigint can not be run inside a transaction' + end + + unless table_exists?(table) + raise "Table #{table} does not exist" + end + + unless column_exists?(table, primary_key) + raise "Column #{primary_key} does not exist on #{table}" + end + + unless column_exists?(table, column) + raise "Column #{column} does not exist on #{table}" + end + + check_trigger_permissions!(table) + + old_column = column_for(table, column) + tmp_column = "#{column}_convert_to_bigint" + + with_lock_retries do + if (column.to_s == primary_key.to_s) || !old_column.null + # If the column to be converted is either a PK or is defined as NOT NULL, + # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow + # That way, we skip the expensive validation step required to add + # a NOT NULL constraint at the end of the process + add_column(table, tmp_column, :bigint, default: old_column.default || 0, null: false) + else + add_column(table, tmp_column, :bigint, default: old_column.default) + end + + install_rename_triggers(table, column, tmp_column) + end + + source_model = Class.new(ActiveRecord::Base) do + include EachBatch + + self.table_name = table + self.inheritance_column = :_type_disabled + end + + queue_background_migration_jobs_by_range_at_intervals( + source_model, + 'CopyColumnUsingBackgroundMigrationJob', + interval, + batch_size: batch_size, + other_job_arguments: [table, primary_key, column, tmp_column, sub_batch_size], + track_jobs: true, + primary_column_name: primary_key + ) + + if perform_background_migration_inline? + # To ensure the schema is up to date immediately we perform the + # migration inline in dev / test environments. + Gitlab::BackgroundMigration.steal('CopyColumnUsingBackgroundMigrationJob') + end + end + # Performs a concurrent column rename when using PostgreSQL. def install_rename_triggers_for_postgresql(trigger, table, old, new) execute <<-EOF.strip_heredoc @@ -996,9 +1165,9 @@ module Gitlab Arel::Nodes::SqlLiteral.new(replace.to_sql) end - def remove_foreign_key_if_exists(*args) - if foreign_key_exists?(*args) - remove_foreign_key(*args) + def remove_foreign_key_if_exists(...) + if foreign_key_exists?(...) + remove_foreign_key(...) end end diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 36073844765..12dcf68da2f 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -100,6 +100,7 @@ module Gitlab end final_delay = 0 + batch_counter = 0 model_class.each_batch(of: batch_size) do |relation, index| start_id, end_id = relation.pluck(Arel.sql("MIN(#{primary_column_name}), MAX(#{primary_column_name})")).first @@ -112,8 +113,17 @@ module Gitlab track_in_database(job_class_name, full_job_arguments) if track_jobs migrate_in(final_delay, job_class_name, full_job_arguments) + + batch_counter += 1 end + duration = initial_delay + delay_interval * batch_counter + say <<~SAY + Scheduled #{batch_counter} #{job_class_name} jobs with a maximum of #{batch_size} records per batch and an interval of #{delay_interval} seconds. + + The migration is expected to take at least #{duration} seconds. Expect all jobs to have completed after #{Time.zone.now + duration}." + SAY + final_delay end diff --git a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb index f367292f4b0..0bc1343acca 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb @@ -32,7 +32,7 @@ module Gitlab return end - partitioned_table.postgres_partitions.each do |partition| + partitioned_table.postgres_partitions.order(:name).each do |partition| partition_index_name = generated_index_name(partition.identifier, options[:name]) partition_options = options.merge(name: partition_index_name) diff --git a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb index 33faa2ef1b0..62dfaeeaae3 100644 --- a/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb +++ b/lib/gitlab/database/postgres_hll/batch_distinct_counter.rb @@ -16,9 +16,9 @@ module Gitlab # Grouped relations are NOT supported yet. # # @example Usage - # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project, :creator_id).estimate_distinct_count + # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project, :creator_id).execute # ::Gitlab::Database::PostgresHllBatchDistinctCount.new(::Project.with_active_services.service_desk_enabled.where(time_period)) - # .estimate_distinct_count( + # .execute( # batch_size: 1_000, # start: ::Project.with_active_services.service_desk_enabled.where(time_period).minimum(:id), # finish: ::Project.with_active_services.service_desk_enabled.where(time_period).maximum(:id) @@ -30,7 +30,6 @@ module Gitlab # for the most of a cases this value is lower. However, if the exact value is necessary other tools has to be used. class BatchDistinctCounter ERROR_RATE = 4.9 # max encountered empirical error rate, used in tests - FALLBACK = -1 MIN_REQUIRED_BATCH_SIZE = 750 SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep MAX_DATA_VOLUME = 4_000_000_000 @@ -38,8 +37,10 @@ module Gitlab # Each query should take < 500ms https://gitlab.com/gitlab-org/gitlab/-/merge_requests/22705 DEFAULT_BATCH_SIZE = 10_000 + ZERO_OFFSET = 1 + BUCKET_ID_MASK = (Buckets::TOTAL_BUCKETS - ZERO_OFFSET).to_s(2) BIT_31_MASK = "B'0#{'1' * 31}'" - BIT_9_MASK = "B'#{'0' * 23}#{'1' * 9}'" + BIT_32_NORMALIZED_BUCKET_ID_MASK = "B'#{'0' * (32 - BUCKET_ID_MASK.size)}#{BUCKET_ID_MASK}'" # @example source_query # SELECT CAST(('X' || md5(CAST(%{column} as text))) as bit(32)) attr_hash_32_bits # FROM %{relation} @@ -48,73 +49,58 @@ module Gitlab # AND %{column} IS NOT NULL BUCKETED_DATA_SQL = <<~SQL WITH hashed_attributes AS (%{source_query}) - SELECT (attr_hash_32_bits & #{BIT_9_MASK})::int AS bucket_num, + SELECT (attr_hash_32_bits & #{BIT_32_NORMALIZED_BUCKET_ID_MASK})::int AS bucket_num, (31 - floor(log(2, min((attr_hash_32_bits & #{BIT_31_MASK})::int))))::int as bucket_hash FROM hashed_attributes GROUP BY 1 SQL - TOTAL_BUCKETS_NUMBER = 512 + WRONG_CONFIGURATION_ERROR = Class.new(ActiveRecord::StatementInvalid) def initialize(relation, column = nil) @relation = relation @column = column || relation.primary_key end - def unwanted_configuration?(finish, batch_size, start) - batch_size <= MIN_REQUIRED_BATCH_SIZE || - (finish - start) >= MAX_DATA_VOLUME || - start > finish - end - - def estimate_distinct_count(batch_size: nil, start: nil, finish: nil) + # Executes counter that iterates over database source and return Gitlab::Database::PostgresHll::Buckets + # that can be used to estimation of number of uniq elements in analysed set + # + # @param batch_size maximal number of rows that will be analysed by single database query + # @param start initial pkey range + # @param finish final pkey range + # @return [Gitlab::Database::PostgresHll::Buckets] HyperLogLog data structure instance that can estimate number of unique elements + def execute(batch_size: nil, start: nil, finish: nil) raise 'BatchCount can not be run inside a transaction' if ActiveRecord::Base.connection.transaction_open? batch_size ||= DEFAULT_BATCH_SIZE - start = actual_start(start) finish = actual_finish(finish) - raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 - return FALLBACK if unwanted_configuration?(finish, batch_size, start) + raise WRONG_CONFIGURATION_ERROR if unwanted_configuration?(start, finish, batch_size) batch_start = start - hll_blob = {} + hll_buckets = Buckets.new while batch_start <= finish begin - hll_blob.merge!(hll_blob_for_batch(batch_start, batch_start + batch_size)) {|_key, old, new| new > old ? new : old } + hll_buckets.merge_hash!(hll_buckets_for_batch(batch_start, batch_start + batch_size)) batch_start += batch_size end sleep(SLEEP_TIME_IN_SECONDS) end - estimate_cardinality(hll_blob) + hll_buckets end private - # arbitrary values that are present in #estimate_cardinality - # are sourced from https://www.sisense.com/blog/hyperloglog-in-pure-sql/ - # article, they are not representing any entity and serves as tune value - # for the whole equation - def estimate_cardinality(hll_blob) - num_zero_buckets = TOTAL_BUCKETS_NUMBER - hll_blob.size - - num_uniques = ( - ((TOTAL_BUCKETS_NUMBER**2) * (0.7213 / (1 + 1.079 / TOTAL_BUCKETS_NUMBER))) / - (num_zero_buckets + hll_blob.values.sum { |bucket_hash| 2**(-1 * bucket_hash)} ) - ).to_i - - if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS_NUMBER - ((0.7213 / (1 + 1.079 / TOTAL_BUCKETS_NUMBER)) * (TOTAL_BUCKETS_NUMBER * - Math.log2(TOTAL_BUCKETS_NUMBER.to_f / num_zero_buckets))) - else - num_uniques - end + def unwanted_configuration?(start, finish, batch_size) + batch_size <= MIN_REQUIRED_BATCH_SIZE || + (finish - start) >= MAX_DATA_VOLUME || + start > finish || start < 0 || finish < 0 end - def hll_blob_for_batch(start, finish) + def hll_buckets_for_batch(start, finish) @relation .connection .execute(BUCKETED_DATA_SQL % { source_query: source_query(start, finish) }) diff --git a/lib/gitlab/database/postgres_hll/buckets.rb b/lib/gitlab/database/postgres_hll/buckets.rb new file mode 100644 index 00000000000..429e823379f --- /dev/null +++ b/lib/gitlab/database/postgres_hll/buckets.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PostgresHll + # Bucket class represent data structure build with HyperLogLog algorithm + # that models data distribution in analysed set. This representation than can be used + # for following purposes + # 1. Estimating number of unique elements that this structure represents + # 2. Merging with other Buckets structure to later estimate number of unique elements in sum of two + # represented data sets + # 3. Serializing Buckets structure to json format, that can be stored in various persistence layers + # + # @example Usage + # ::Gitlab::Database::PostgresHll::Buckets.new(141 => 1, 56 => 1).estimated_distinct_count + # ::Gitlab::Database::PostgresHll::Buckets.new(141 => 1, 56 => 1).merge_hash!(141 => 1, 56 => 5).estimated_distinct_count + # ::Gitlab::Database::PostgresHll::Buckets.new(141 => 1, 56 => 1).to_json + + # @note HyperLogLog is an PROBABILISTIC algorithm that ESTIMATES distinct count of given attribute value for supplied relation + # Like all probabilistic algorithm is has ERROR RATE margin, that can affect values, + # for given implementation no higher value was reported (https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45673#accuracy-estimation) than 5.3% + # for the most of a cases this value is lower. However, if the exact value is necessary other tools has to be used. + class Buckets + TOTAL_BUCKETS = 512 + + def initialize(buckets = {}) + @buckets = buckets + end + + # Based on HyperLogLog structure estimates number of unique elements in analysed set. + # + # @return [Float] Estimate number of unique elements + def estimated_distinct_count + @estimated_distinct_count ||= estimate_cardinality + end + + # Updates instance underlying HyperLogLog structure by merging it with other HyperLogLog structure + # + # @param other_buckets_hash hash with HyperLogLog structure representation + def merge_hash!(other_buckets_hash) + buckets.merge!(other_buckets_hash) {|_key, old, new| new > old ? new : old } + end + + # Serialize instance underlying HyperLogLog structure to JSON format, that can be stored in various persistence layers + # + # @return [String] HyperLogLog data structure serialized to JSON + def to_json(_ = nil) + buckets.to_json + end + + private + + attr_accessor :buckets + + # arbitrary values that are present in #estimate_cardinality + # are sourced from https://www.sisense.com/blog/hyperloglog-in-pure-sql/ + # article, they are not representing any entity and serves as tune value + # for the whole equation + def estimate_cardinality + num_zero_buckets = TOTAL_BUCKETS - buckets.size + + num_uniques = ( + ((TOTAL_BUCKETS**2) * (0.7213 / (1 + 1.079 / TOTAL_BUCKETS))) / + (num_zero_buckets + buckets.values.sum { |bucket_hash| 2**(-1 * bucket_hash)} ) + ).to_i + + if num_zero_buckets > 0 && num_uniques < 2.5 * TOTAL_BUCKETS + ((0.7213 / (1 + 1.079 / TOTAL_BUCKETS)) * (TOTAL_BUCKETS * + Math.log2(TOTAL_BUCKETS.to_f / num_zero_buckets))) + else + num_uniques + end + end + end + end + end +end diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 832f7438cf9..0cfad690283 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -8,9 +8,9 @@ module Gitlab # candidate_indexes: Array of Gitlab::Database::PostgresIndex def self.perform(candidate_indexes, how_many: DEFAULT_INDEXES_PER_INVOCATION) - indexes = IndexSelection.new(candidate_indexes).take(how_many) - - Coordinator.new(indexes).perform + IndexSelection.new(candidate_indexes).take(how_many).each do |index| + Coordinator.new(index).perform + end end def self.candidate_indexes diff --git a/lib/gitlab/database/reindexing/coordinator.rb b/lib/gitlab/database/reindexing/coordinator.rb index 0957f43e166..7a7d17ca196 100644 --- a/lib/gitlab/database/reindexing/coordinator.rb +++ b/lib/gitlab/database/reindexing/coordinator.rb @@ -12,26 +12,44 @@ module Gitlab # statement timeouts). TIMEOUT_PER_ACTION = 1.day - attr_reader :indexes + attr_reader :index, :notifier - def initialize(indexes) - @indexes = indexes + def initialize(index, notifier = GrafanaNotifier.new) + @index = index + @notifier = notifier end def perform - indexes.each do |index| - # This obtains a global lease such that there's - # only one live reindexing process at a time. - try_obtain_lease do - ReindexAction.keep_track_of(index) do - ConcurrentReindex.new(index).perform - end + # This obtains a global lease such that there's + # only one live reindexing process at a time. + try_obtain_lease do + action = ReindexAction.create_for(index) + + with_notifications(action) do + perform_for(index, action) end end end private + def with_notifications(action) + notifier.notify_start(action) + yield + ensure + notifier.notify_end(action) + end + + def perform_for(index, action) + ConcurrentReindex.new(index).perform + rescue + action.state = :failed + + raise + ensure + action.finish + end + def lease_timeout TIMEOUT_PER_ACTION end diff --git a/lib/gitlab/database/reindexing/grafana_notifier.rb b/lib/gitlab/database/reindexing/grafana_notifier.rb new file mode 100644 index 00000000000..b1e5ecb9ade --- /dev/null +++ b/lib/gitlab/database/reindexing/grafana_notifier.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Reindexing + # This can be used to send annotations for reindexing to a Grafana API + class GrafanaNotifier + def initialize(api_key = ENV['GITLAB_GRAFANA_API_KEY'], api_url = ENV['GITLAB_GRAFANA_API_URL'], additional_tag = ENV['GITLAB_REINDEXING_GRAFANA_TAG'] || Rails.env) + @api_key = api_key + @api_url = api_url + @additional_tag = additional_tag + end + + def notify_start(action) + return unless enabled? + + payload = base_payload(action).merge( + text: "Started reindexing of #{action.index.name} on #{action.index.tablename}" + ) + + annotate(payload) + end + + def notify_end(action) + return unless enabled? + + payload = base_payload(action).merge( + text: "Finished reindexing of #{action.index.name} on #{action.index.tablename} (#{action.state})", + timeEnd: (action.action_end.utc.to_f * 1000).to_i, + isRegion: true + ) + + annotate(payload) + end + + private + + def base_payload(action) + { + time: (action.action_start.utc.to_f * 1000).to_i, + tags: ['reindex', @additional_tag, action.index.tablename, action.index.name].compact + } + end + + def annotate(payload) + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer #{@api_key}" + } + + success = Gitlab::HTTP.post("#{@api_url}/api/annotations", body: payload.to_json, headers: headers, allow_local_requests: true).success? + + log_error("Response code #{response.code}") unless success + + success + rescue => err + log_error(err) + + false + end + + def log_error(err) + Gitlab::AppLogger.warn("Unable to notify Grafana from #{self.class}: #{err}") + end + + def enabled? + !(@api_url.blank? || @api_key.blank?) + end + end + end + end +end diff --git a/lib/gitlab/database/reindexing/reindex_action.rb b/lib/gitlab/database/reindexing/reindex_action.rb index 8c59cffe5fb..7e58201889f 100644 --- a/lib/gitlab/database/reindexing/reindex_action.rb +++ b/lib/gitlab/database/reindexing/reindex_action.rb @@ -14,27 +14,23 @@ module Gitlab scope :recent, -> { where(state: :finished).where('action_end > ?', Time.zone.now - RECENT_THRESHOLD) } - def self.keep_track_of(index, &block) - action = create!( + def self.create_for(index) + create!( index_identifier: index.identifier, action_start: Time.zone.now, ondisk_size_bytes_start: index.ondisk_size_bytes, bloat_estimate_bytes_start: index.bloat_size ) + end - yield - - action.state = :finished - rescue - action.state = :failed - raise - ensure + def finish index.reload # rubocop:disable Cop/ActiveRecordAssociationReload - action.action_end = Time.zone.now - action.ondisk_size_bytes_end = index.ondisk_size_bytes + self.state = :finished unless failed? + self.action_end = Time.zone.now + self.ondisk_size_bytes_end = index.ondisk_size_bytes - action.save! + save! end end end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index b1093b2fca4..d1ada8c723e 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -75,7 +75,7 @@ module Gitlab if response # In the add_prometheus_manual_configuration method, the Prometheus - # listen_address config is saved as an api_url in the PrometheusService + # server_address config is saved as an api_url in the PrometheusService # model. There are validates hooks in the PrometheusService model that # check if the project associated with the PrometheusService is the # self_monitoring project. It checks @@ -105,7 +105,7 @@ module Gitlab def add_prometheus_manual_configuration(result) return success(result) unless prometheus_enabled? - return success(result) unless prometheus_listen_address.present? + return success(result) unless prometheus_server_address.present? service = result[:project].find_or_initialize_service('prometheus') @@ -132,8 +132,8 @@ module Gitlab ::Gitlab::Prometheus::Internal.prometheus_enabled? end - def prometheus_listen_address - ::Gitlab::Prometheus::Internal.listen_address + def prometheus_server_address + ::Gitlab::Prometheus::Internal.server_address end def docs_path @@ -152,13 +152,13 @@ module Gitlab } end - def internal_prometheus_listen_address_uri + def internal_prometheus_server_address_uri ::Gitlab::Prometheus::Internal.uri end def prometheus_service_attributes { - api_url: internal_prometheus_listen_address_uri, + api_url: internal_prometheus_server_address_uri, manual_configuration: true, active: true } diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index af9140215f0..98ed2400d82 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -8,9 +8,9 @@ module Gitlab # SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze - attr_reader :line_code, :old_pos, :new_pos + attr_reader :line_code attr_writer :rich_text - attr_accessor :text, :index, :type + attr_accessor :text, :index, :type, :old_pos, :new_pos def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil) @text, @type, @index = text, type, index diff --git a/lib/gitlab/diff/position.rb b/lib/gitlab/diff/position.rb index e43f301c280..74c33c46598 100644 --- a/lib/gitlab/diff/position.rb +++ b/lib/gitlab/diff/position.rb @@ -19,6 +19,7 @@ module Gitlab :height, :x, :y, + :line_range, :position_type, to: :formatter # A position can belong to a text line or to an image coordinate @@ -167,6 +168,12 @@ module Gitlab end end + def multiline? + return unless on_text? && line_range + + line_range['start'] != line_range['end'] + end + private def find_diff_file(repository) diff --git a/lib/gitlab/email/handler.rb b/lib/gitlab/email/handler.rb index 1b8421d34f3..e71ea154355 100644 --- a/lib/gitlab/email/handler.rb +++ b/lib/gitlab/email/handler.rb @@ -11,6 +11,7 @@ module Gitlab [ CreateNoteHandler, CreateIssueHandler, + CreateNoteOnIssuableHandler, UnsubscribeHandler, CreateMergeRequestHandler, ServiceDeskHandler diff --git a/lib/gitlab/email/handler/create_note_on_issuable_handler.rb b/lib/gitlab/email/handler/create_note_on_issuable_handler.rb new file mode 100644 index 00000000000..aed3647744a --- /dev/null +++ b/lib/gitlab/email/handler/create_note_on_issuable_handler.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require 'gitlab/email/handler/base_handler' + +# Handles comment creation emails when sent/forwarded by an authorized +# user. Attachments are allowed. Quoted material is _not_ stripped, just like +# create issue emails +# Supports these formats: +# incoming+gitlab-org-gitlab-ce-20-Author_Token12345678-issue-34@incoming.gitlab.com +module Gitlab + module Email + module Handler + class CreateNoteOnIssuableHandler < BaseHandler + include ReplyProcessing + + attr_reader :issuable_iid + + HANDLER_REGEX = /\A#{HANDLER_ACTION_BASE_REGEX}-(?<incoming_email_token>.+)-issue-(?<issuable_iid>\d+)\z/.freeze + + def initialize(mail, mail_key) + super(mail, mail_key) + + if (matched = HANDLER_REGEX.match(mail_key.to_s)) + @project_slug = matched[:project_slug] + @project_id = matched[:project_id]&.to_i + @incoming_email_token = matched[:incoming_email_token] + @issuable_iid = matched[:issuable_iid]&.to_i + end + end + + def can_handle? + incoming_email_token && project_id && issuable_iid + end + + def execute + raise ProjectNotFound unless project + + validate_permission!(:create_note) + + raise NoteableNotFoundError unless noteable + raise EmptyEmailError if message_including_reply.blank? + + verify_record!( + record: create_note, + invalid_exception: InvalidNoteError, + record_name: 'comment') + end + + def metrics_event + :receive_email_create_note_issuable + end + + def noteable + return unless issuable_iid + + @noteable ||= project&.issues&.find_by_iid(issuable_iid) + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def author + @author ||= User.find_by(incoming_email_token: incoming_email_token) + end + # rubocop: enable CodeReuse/ActiveRecord + + def create_note + Notes::CreateService.new(project, author, note_params).execute + end + + def note_params + { + noteable_type: noteable.class.to_s, + noteable_id: noteable.id, + note: message_including_reply + } + end + end + end + end +end diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 0bbe3980f67..f66e8a8794f 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -68,7 +68,7 @@ module Gitlab end def valid_project_key?(project, slug) - project.present? && slug == project.full_path_slug && Feature.enabled?(:service_desk_custom_address, project, default_enabled: true) + project.present? && slug == project.full_path_slug end def create_issue! diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index a5ace2be773..1a8e5aaf07a 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -111,8 +111,8 @@ module Gitlab private def before_send(event, hint) - event = add_context_from_exception_type(event, hint) - event = custom_fingerprinting(event, hint) + inject_context_for_exception(event, hint[:exception]) + custom_fingerprinting(event, hint[:exception]) event end @@ -123,7 +123,6 @@ module Gitlab end extra = sanitize_request_parameters(extra) - inject_sql_query_into_extra(exception, extra) if sentry && Raven.configuration.server Raven.capture_exception(exception, tags: default_tags, extra: extra) @@ -150,12 +149,6 @@ module Gitlab filter.filter(parameters) end - def inject_sql_query_into_extra(exception, extra) - return unless exception.is_a?(ActiveRecord::StatementInvalid) - - extra[:sql] = PgQuery.normalize(exception.sql.to_s) - end - def sentry_dsn return unless Rails.env.production? || Rails.env.development? return unless Gitlab.config.sentry.enabled @@ -183,31 +176,21 @@ module Gitlab {} end - # Debugging for https://gitlab.com/gitlab-org/gitlab-foss/issues/57727 - def add_context_from_exception_type(event, hint) - if ActiveModel::MissingAttributeError === hint[:exception] - columns_hash = ActiveRecord::Base - .connection - .schema_cache - .instance_variable_get(:@columns_hash) - .transform_values { |v| v.map(&:first) } - - event.extra.merge!(columns_hash) - end - - event - end - # Group common, mostly non-actionable exceptions by type and message, # rather than cause - def custom_fingerprinting(event, hint) - ex = hint[:exception] - + def custom_fingerprinting(event, ex) return event unless CUSTOM_FINGERPRINTING.include?(ex.class.name) event.fingerprint = [ex.class.name, ex.message] + end - event + def inject_context_for_exception(event, ex) + case ex + when ActiveRecord::StatementInvalid + event.extra[:sql] = PgQuery.normalize(ex.sql.to_s) + else + inject_context_for_exception(event, ex.cause) if ex.cause.present? + end end end end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 94523813662..196203211ed 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -87,6 +87,24 @@ module Gitlab }, invite_members_empty_project_version_a: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersEmptyProjectVersionA' + }, + trial_during_signup: { + tracking_category: 'Growth::Conversion::Experiment::TrialDuringSignup' + }, + ci_syntax_templates: { + tracking_category: 'Growth::Activation::Experiment::CiSyntaxTemplates' + }, + pipelines_empty_state: { + tracking_category: 'Growth::Activation::Experiment::PipelinesEmptyState' + }, + invite_members_new_dropdown: { + tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' + }, + show_trial_status_in_sidebar: { + tracking_category: 'Growth::Conversion::Experiment::ShowTrialStatusInSidebar' + }, + trial_onboarding_issues: { + tracking_category: 'Growth::Conversion::Experiment::TrialOnboardingIssues' } }.freeze diff --git a/lib/gitlab/experimentation/controller_concern.rb b/lib/gitlab/experimentation/controller_concern.rb index c85d3f4eee6..e43f3c8c007 100644 --- a/lib/gitlab/experimentation/controller_concern.rb +++ b/lib/gitlab/experimentation/controller_concern.rb @@ -15,7 +15,7 @@ module Gitlab included do before_action :set_experimentation_subject_id_cookie, unless: :dnt_enabled? - helper_method :experiment_enabled?, :experiment_tracking_category_and_group + helper_method :experiment_enabled?, :experiment_tracking_category_and_group, :tracking_label end def set_experimentation_subject_id_cookie @@ -130,7 +130,10 @@ module Gitlab end def forced_enabled?(experiment_key) - params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s + return true if params.has_key?(:force_experiment) && params[:force_experiment] == experiment_key.to_s + return false if cookies[:force_experiment].blank? + + cookies[:force_experiment].to_s.split(',').any? { |experiment| experiment.strip == experiment_key.to_s } end def tracking_label(subject) diff --git a/lib/gitlab/experimentation/experiment.rb b/lib/gitlab/experimentation/experiment.rb index e594c3bedeb..36cd673a38f 100644 --- a/lib/gitlab/experimentation/experiment.rb +++ b/lib/gitlab/experimentation/experiment.rb @@ -3,17 +3,21 @@ module Gitlab module Experimentation class Experiment + FEATURE_FLAG_SUFFIX = "_experiment_percentage" + attr_reader :key, :tracking_category, :use_backwards_compatible_subject_index def initialize(key, **params) @key = key @tracking_category = params[:tracking_category] @use_backwards_compatible_subject_index = params[:use_backwards_compatible_subject_index] - - @experiment_percentage = Feature.get(:"#{key}_experiment_percentage").percentage_of_time_value # rubocop:disable Gitlab/AvoidFeatureGet end def active? + # TODO: just touch a feature flag + # Temporary change, we will change `experiment_percentage` in future to `Feature.enabled? + Feature.enabled?(feature_flag_name, type: :experiment, default_enabled: :yaml) + ::Gitlab.dev_env_or_com? && experiment_percentage > 0 end @@ -25,7 +29,17 @@ module Gitlab private - attr_reader :experiment_percentage + def experiment_percentage + feature_flag.percentage_of_time_value + end + + def feature_flag + Feature.get(feature_flag_name) # rubocop:disable Gitlab/AvoidFeatureGet + end + + def feature_flag_name + :"#{key}#{FEATURE_FLAG_SUFFIX}" + end end end end diff --git a/lib/gitlab/faraday.rb b/lib/gitlab/faraday.rb new file mode 100644 index 00000000000..f92392ec1a9 --- /dev/null +++ b/lib/gitlab/faraday.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Gitlab + module Faraday + ::Faraday::Request.register_middleware(gitlab_error_callback: -> { ::Gitlab::Faraday::ErrorCallback }) + end +end diff --git a/lib/gitlab/faraday/error_callback.rb b/lib/gitlab/faraday/error_callback.rb new file mode 100644 index 00000000000..f99be5b4d04 --- /dev/null +++ b/lib/gitlab/faraday/error_callback.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Faraday + # Simple Faraday Middleware that catches any error risen during the request and run the configured callback. + # (https://lostisland.github.io/faraday/middleware/) + # + # By default, a no op callback is setup. + # + # Note that the error is not swallowed: it will be rerisen again. In that regard, this callback acts more + # like an error spy than anything else. + # + # The callback has access to the request `env` and the exception instance. For more details, see + # https://lostisland.github.io/faraday/middleware/custom + # + # Faraday.new do |conn| + # conn.request( + # :error_callback, + # callback: -> (env, exception) { Rails.logger.debug("Error #{exception.class.name} when trying to contact #{env[:url]}" ) } + # ) + # conn.adapter(:net_http) + # end + class ErrorCallback < ::Faraday::Middleware + def initialize(app, options = nil) + super(app) + @options = ::Gitlab::Faraday::ErrorCallback::Options.from(options) # rubocop: disable CodeReuse/ActiveRecord + end + + def call(env) + @app.call(env) + rescue => e + @options.callback&.call(env, e) + + raise + end + + class Options < ::Faraday::Options.new(:callback) + def callback + self[:callback] + end + end + end + end +end diff --git a/lib/gitlab/git/changed_path.rb b/lib/gitlab/git/changed_path.rb new file mode 100644 index 00000000000..033779466f6 --- /dev/null +++ b/lib/gitlab/git/changed_path.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class ChangedPath + attr_reader :status, :path + + def initialize(status:, path:) + @status = status + @path = path + end + + def new_file? + status == :ADDED + end + end + end +end diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 8df4bc3de05..19462e6cb02 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -13,7 +13,7 @@ module Gitlab def self.default_limits(project: nil) if Feature.enabled?(:increased_diff_limits, project) - { max_files: 200, max_lines: 7500 } + { max_files: 300, max_lines: 10000 } else { max_files: 100, max_lines: 5000 } end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index f6601379202..e316d52ac05 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -801,7 +801,8 @@ module Gitlab # forced - should we use --force flag? # no_tags - should we use --no-tags flag? # prune - should we use --prune flag? - def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false, prune: true) + # check_tags_changed - should we ask gitaly to calculate whether any tags changed? + def fetch_remote(remote, ssh_auth: nil, forced: false, no_tags: false, prune: true, check_tags_changed: false) wrapped_gitaly_errors do gitaly_repository_client.fetch_remote( remote, @@ -809,6 +810,7 @@ module Gitlab forced: forced, no_tags: no_tags, prune: prune, + check_tags_changed: check_tags_changed, timeout: GITLAB_PROJECTS_TIMEOUT ) end diff --git a/lib/gitlab/git/wiki_page_version.rb b/lib/gitlab/git/wiki_page_version.rb index 475a9d4d1b9..efe39fa852c 100644 --- a/lib/gitlab/git/wiki_page_version.rb +++ b/lib/gitlab/git/wiki_page_version.rb @@ -10,7 +10,12 @@ module Gitlab @format = format end - delegate :message, :sha, :id, :author_name, :authored_date, to: :commit + delegate :message, :sha, :id, :author_name, :author_email, :authored_date, to: :commit + + def author_url + user = ::User.find_by_any_email(author_email) + user.nil? ? "mailto:#{author_email}" : Gitlab::UrlBuilder.build(user) + end end end end diff --git a/lib/gitlab/git_access_snippet.rb b/lib/gitlab/git_access_snippet.rb index 854bf6e9c9e..88a75f72840 100644 --- a/lib/gitlab/git_access_snippet.rb +++ b/lib/gitlab/git_access_snippet.rb @@ -30,7 +30,10 @@ module Gitlab def check(cmd, changes) check_snippet_accessibility! - super + super.tap do |_| + # Ensure HEAD points to the default branch in case it is not master + snippet.change_head_to_default_branch + end end override :download_ability @@ -56,7 +59,7 @@ module Gitlab # TODO: Investigate if expanding actor/authentication types are needed. # https://gitlab.com/gitlab-org/gitlab/issues/202190 if actor && !allowed_actor? - raise ForbiddenError, ERROR_MESSAGES[:authentication_mechanism] + raise ForbiddenError, error_message(:authentication_mechanism) end super @@ -68,14 +71,18 @@ module Gitlab override :check_push_access! def check_push_access! - raise ForbiddenError, ERROR_MESSAGES[:update_snippet] unless user + raise ForbiddenError, error_message(:update_snippet) unless user + + if snippet&.repository_read_only? + raise ForbiddenError, error_message(:read_only) + end check_change_access! end def check_snippet_accessibility! if snippet.blank? - raise NotFoundError, ERROR_MESSAGES[:snippet_not_found] + raise NotFoundError, error_message(:snippet_not_found) end end @@ -91,14 +98,14 @@ module Gitlab passed = guest_can_download_code? || user_can_download_code? unless passed - raise ForbiddenError, ERROR_MESSAGES[:read_snippet] + raise ForbiddenError, error_message(:read_snippet) end end override :check_change_access! def check_change_access! unless user_can_push? - raise ForbiddenError, ERROR_MESSAGES[:update_snippet] + raise ForbiddenError, error_message(:update_snippet) end check_size_before_push! diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index e1324530412..31734abe77f 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -215,12 +215,16 @@ module Gitlab 'client_name' => CLIENT_NAME } + context_data = Labkit::Context.current&.to_h + feature_stack = Thread.current[:gitaly_feature_stack] feature = feature_stack && feature_stack[0] metadata['call_site'] = feature.to_s if feature metadata['gitaly-servers'] = address_metadata(remote_storage) if remote_storage metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id metadata['gitaly-session-id'] = session_id + metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil) + metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil) metadata.merge!(Feature::Gitaly.server_feature_flags) deadline_info = request_deadline(timeout) diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 599bce176c9..ea940150941 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -225,7 +225,7 @@ module Gitlab response = GitalyClient.call(@repository.storage, :diff_service, :find_changed_paths, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| msg.paths.map do |path| - OpenStruct.new( + Gitlab::Git::ChangedPath.new( status: path.status, path: EncodingHelper.encode!(path.path) ) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index e41a406ebd3..bd450249355 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -70,10 +70,11 @@ module Gitlab end.join end - def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:, prune: true) + def fetch_remote(remote, ssh_auth:, forced:, no_tags:, timeout:, prune: true, check_tags_changed: false) request = Gitaly::FetchRemoteRequest.new( repository: @gitaly_repo, remote: remote, force: forced, - no_tags: no_tags, timeout: timeout, no_prune: !prune + no_tags: no_tags, timeout: timeout, no_prune: !prune, + check_tags_changed: check_tags_changed ) if ssh_auth&.ssh_mirror_url? diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 7ae91912b8a..1401c92a44e 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -56,7 +56,7 @@ module Gitlab # The initial fetch can bring in lots of loose refs and objects. # Running a `git gc` will make importing pull requests faster. - Projects::HousekeepingService.new(project, :gc).execute + Repositories::HousekeepingService.new(project, :gc).execute true rescue Gitlab::Git::Repository::NoRepository, Gitlab::Shell::Error => e diff --git a/lib/gitlab/gitpod.rb b/lib/gitlab/gitpod.rb deleted file mode 100644 index e35fb8fed02..00000000000 --- a/lib/gitlab/gitpod.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class Gitpod - class << self - def feature_available? - # The gitpod_bundle feature could be conditionally applied, so check if `!off?` - !feature.off? || feature_enabled? - end - - def feature_enabled?(actor = nil) - Feature.enabled?(:gitpod, actor, default_enabled: true) - end - - def feature_and_settings_enabled?(actor = nil) - feature_enabled?(actor) && Gitlab::CurrentSettings.gitpod_enabled - end - - private - - def feature - Feature.get(:gitpod) # rubocop:disable Gitlab/AvoidFeatureGet - end - end - end -end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 362da8ea53e..0ba535b500e 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -4,7 +4,6 @@ module Gitlab module GonHelper - include StartupCssHelper include WebpackHelper def add_gon_variables @@ -48,9 +47,7 @@ module Gitlab push_frontend_feature_flag(:snippets_binary_blob, default_enabled: false) push_frontend_feature_flag(:usage_data_api, default_enabled: true) push_frontend_feature_flag(:security_auto_fix, default_enabled: false) - - # Startup CSS feature is a special one as it can be enabled by means of cookies and params - gon.push({ features: { 'startupCss' => use_startup_css? } }, true) + push_frontend_feature_flag(:gl_tooltips, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/batch_key.rb b/lib/gitlab/graphql/batch_key.rb new file mode 100644 index 00000000000..51203af5a43 --- /dev/null +++ b/lib/gitlab/graphql/batch_key.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + class BatchKey + attr_reader :object + delegate :hash, to: :object + + def initialize(object, lookahead = nil, object_name: nil) + @object = object + @lookahead = lookahead + @object_name = object_name + end + + def requires?(path) + return false unless @lookahead + return false unless path.present? + + field = path.pop + + path + .reduce(@lookahead) { |q, f| q.selection(f) } + .selects?(field) + end + + def eql?(other) + other.is_a?(self.class) && object == other.object + end + alias_method :==, :eql? + + def method_missing(method_name, *args, **kwargs) + return @object if method_name.to_sym == @object_name + return @object.public_send(method_name) if args.empty? && kwargs.empty? # rubocop: disable GitlabSecurity/PublicSend + + super + end + end + end +end diff --git a/lib/gitlab/graphql/lazy.rb b/lib/gitlab/graphql/lazy.rb index 54013cf4790..3563504226c 100644 --- a/lib/gitlab/graphql/lazy.rb +++ b/lib/gitlab/graphql/lazy.rb @@ -17,6 +17,14 @@ module Gitlab self.class.new { yield force } end + def catch(error_class = StandardError, &block) + self.class.new do + force + rescue error_class => e + yield e + end + end + # Force evaluation of a (possibly) lazy value def self.force(value) case value diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index 2ad8d2f7ab7..f95c91c5706 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -67,9 +67,14 @@ module Gitlab # next page true elsif first - # If we count the number of requested items plus one (`limit_value + 1`), - # then if we get `limit_value + 1` then we know there is a next page - relation_count(set_limit(sliced_nodes, limit_value + 1)) == limit_value + 1 + case sliced_nodes + when Array + sliced_nodes.size > limit_value + else + # If we count the number of requested items plus one (`limit_value + 1`), + # then if we get `limit_value + 1` then we know there is a next page + relation_count(set_limit(sliced_nodes, limit_value + 1)) == limit_value + 1 + end else false end @@ -157,8 +162,8 @@ module Gitlab list = OrderInfo.build_order_list(items) - if loaded?(items) - @order_list = list.presence || [items.primary_key] + if loaded?(items) && !before.present? && !after.present? + @order_list = list.presence || [OrderInfo.new(items.primary_key)] # already sorted, or trivially sorted next items if list.present? || items.size <= 1 @@ -194,7 +199,7 @@ module Gitlab ordering = { 'id' => node[:id].to_s } order_list.each do |field| - field_name = field.attribute_name + field_name = field.try(:attribute_name) || field field_value = node[field_name] ordering[field_name] = if field_value.is_a?(Time) field_value.strftime('%Y-%m-%d %H:%M:%S.%N %Z') diff --git a/lib/gitlab/graphql/pagination/keyset/query_builder.rb b/lib/gitlab/graphql/pagination/keyset/query_builder.rb index 331981ce723..29169449843 100644 --- a/lib/gitlab/graphql/pagination/keyset/query_builder.rb +++ b/lib/gitlab/graphql/pagination/keyset/query_builder.rb @@ -40,7 +40,10 @@ module Gitlab # "issues"."id" > 500 # def conditions - attr_values = order_list.map { |field| decoded_cursor[field.attribute_name] } + attr_values = order_list.map do |field| + name = field.try(:attribute_name) || field + decoded_cursor[name] + end if order_list.count == 1 && attr_values.first.nil? raise Gitlab::Graphql::Errors::ArgumentError.new('Before/after cursor invalid: `nil` was provided as only sortable value') diff --git a/lib/gitlab/graphql/queries.rb b/lib/gitlab/graphql/queries.rb new file mode 100644 index 00000000000..de971743490 --- /dev/null +++ b/lib/gitlab/graphql/queries.rb @@ -0,0 +1,286 @@ +# frozen_string_literal: true + +require 'find' + +module Gitlab + module Graphql + module Queries + IMPORT_RE = /^#\s*import "(?<path>[^"]+)"$/m.freeze + EE_ELSE_CE = /^ee_else_ce/.freeze + HOME_RE = /^~/.freeze + HOME_EE = %r{^ee/}.freeze + DOTS_RE = %r{^(\.\./)+}.freeze + DOT_RE = %r{^\./}.freeze + IMPLICIT_ROOT = %r{^app/}.freeze + CONN_DIRECTIVE = /@connection\(key: "\w+"\)/.freeze + + class WrappedError + delegate :message, to: :@error + + def initialize(error) + @error = error + end + + def path + [] + end + end + + class FileNotFound + def initialize(file) + @file = file + end + + def message + "File not found: #{@file}" + end + + def path + [] + end + end + + # We need to re-write queries to remove all @client fields. Ideally we + # would do that as a source-to-source transformation of the AST, but doing it using a + # printer is much simpler. + class ClientFieldRedactor < GraphQL::Language::Printer + attr_reader :fields_printed, :skipped_arguments, :printed_arguments, :used_fragments + + def initialize(skips = true) + @skips = skips + @fields_printed = 0 + @in_operation = false + @skipped_arguments = [].to_set + @printed_arguments = [].to_set + @used_fragments = [].to_set + @skipped_fragments = [].to_set + @used_fragments = [].to_set + end + + def print_variable_identifier(variable_identifier) + @printed_arguments << variable_identifier.name + super + end + + def print_fragment_spread(fragment_spread, indent: "") + @used_fragments << fragment_spread.name + super + end + + def print_operation_definition(op, indent: "") + @in_operation = true + out = +"#{indent}#{op.operation_type}" + out << " #{op.name}" if op.name + + # Do these first, so that we detect any skipped arguments + dirs = print_directives(op.directives) + sels = print_selections(op.selections, indent: indent) + + # remove variable definitions only used in skipped (client) fields + vars = op.variables.reject do |v| + @skipped_arguments.include?(v.name) && !@printed_arguments.include?(v.name) + end + + if vars.any? + out << "(#{vars.map { |v| print_variable_definition(v) }.join(", ")})" + end + + out + dirs + sels + ensure + @in_operation = false + end + + def print_field(field, indent: '') + if skips? && field.directives.any? { |d| d.name == 'client' } + skipped = self.class.new(false) + + skipped.print_node(field) + @skipped_fragments |= skipped.used_fragments + @skipped_arguments |= skipped.printed_arguments + + return '' + end + + ret = super + + @fields_printed += 1 if @in_operation && ret != '' + + ret + end + + def print_fragment_definition(fragment_def, indent: "") + if skips? && @skipped_fragments.include?(fragment_def.name) && !@used_fragments.include?(fragment_def.name) + return '' + end + + super + end + + def skips? + @skips + end + end + + class Definition + attr_reader :file, :imports + + def initialize(path, fragments) + @file = path + @fragments = fragments + @imports = [] + @errors = [] + @ee_else_ce = [] + end + + def text(mode: :ce) + qs = [query] + all_imports(mode: mode).uniq.sort.map { |p| fragment(p).query } + t = qs.join("\n\n").gsub(/\n\n+/, "\n\n") + + return t unless /@client/.match?(t) + + doc = ::GraphQL.parse(t) + printer = ClientFieldRedactor.new + redacted = doc.dup.to_query_string(printer: printer) + + return redacted if printer.fields_printed > 0 + end + + def query + return @query if defined?(@query) + + # CONN_DIRECTIVEs are purely client-side constructs + @query = File.read(file).gsub(CONN_DIRECTIVE, '').gsub(IMPORT_RE) do + path = $~[:path] + + if EE_ELSE_CE.match?(path) + @ee_else_ce << path.gsub(EE_ELSE_CE, '') + else + @imports << fragment_path(path) + end + + '' + end + rescue Errno::ENOENT + @errors << FileNotFound.new(file) + @query = nil + end + + def all_imports(mode: :ce) + return [] if query.nil? + + home = mode == :ee ? @fragments.home_ee : @fragments.home + eithers = @ee_else_ce.map { |p| home + p } + + (imports + eithers).flat_map { |p| [p] + @fragments.get(p).all_imports(mode: mode) } + end + + def all_errors + return @errors.to_set if query.nil? + + paths = imports + @ee_else_ce.flat_map { |p| [@fragments.home + p, @fragments.home_ee + p] } + + paths.map { |p| fragment(p).all_errors }.reduce(@errors.to_set) { |a, b| a | b } + end + + def validate(schema) + return [:client_query, []] if query.present? && text.nil? + + errs = all_errors.presence || schema.validate(text) + if @ee_else_ce.present? + errs += schema.validate(text(mode: :ee)) + end + + [:validated, errs] + rescue ::GraphQL::ParseError => e + [:validated, [WrappedError.new(e)]] + end + + private + + def fragment(path) + @fragments.get(path) + end + + def fragment_path(import_path) + frag_path = import_path.gsub(HOME_RE, @fragments.home) + frag_path = frag_path.gsub(HOME_EE, @fragments.home_ee + '/') + frag_path = frag_path.gsub(DOT_RE) do + Pathname.new(file).parent.to_s + '/' + end + frag_path = frag_path.gsub(DOTS_RE) do |dots| + rel_dir(dots.split('/').count) + end + frag_path = frag_path.gsub(IMPLICIT_ROOT) do + (Rails.root / 'app').to_s + '/' + end + + frag_path + end + + def rel_dir(n_steps_up) + path = Pathname.new(file).parent + while n_steps_up > 0 + path = path.parent + n_steps_up -= 1 + end + + path.to_s + '/' + end + end + + class Fragments + def initialize(root, dir = 'app/assets/javascripts') + @root = root + @store = {} + @dir = dir + end + + def home + @home ||= (@root / @dir).to_s + end + + def home_ee + @home_ee ||= (@root / 'ee' / @dir).to_s + end + + def get(frag_path) + @store[frag_path] ||= Definition.new(frag_path, self) + end + end + + def self.find(root) + definitions = [] + + ::Find.find(root.to_s) do |path| + definitions << Definition.new(path, fragments) if query?(path) + end + + definitions + rescue Errno::ENOENT + [] # root does not exist + end + + def self.fragments + @fragments ||= Fragments.new(Rails.root) + end + + def self.all + ['.', 'ee'].flat_map do |prefix| + find(Rails.root / prefix / 'app/assets/javascripts') + end + end + + def self.known_failure?(path) + @known_failures ||= YAML.safe_load(File.read(Rails.root.join('config', 'known_invalid_graphql_queries.yml'))) + + @known_failures.fetch('filenames', []).any? { |known_failure| path.to_s.ends_with?(known_failure) } + end + + def self.query?(path) + path.ends_with?('.graphql') && + !path.ends_with?('.fragment.graphql') && + !path.ends_with?('typedefs.graphql') + end + end + end +end diff --git a/lib/gitlab/hashed_storage/rake_helper.rb b/lib/gitlab/hashed_storage/rake_helper.rb index 7965f165683..d3468569e5e 100644 --- a/lib/gitlab/hashed_storage/rake_helper.rb +++ b/lib/gitlab/hashed_storage/rake_helper.rb @@ -65,6 +65,7 @@ module Gitlab def self.projects_list(relation_name, relation) listing(relation_name, relation.with_route) do |project| $stdout.puts " - #{project.full_path} (id: #{project.id})".color(:red) + $stdout.puts " #{project.repository.disk_path}" end end @@ -92,6 +93,37 @@ module Gitlab end end # rubocop: enable CodeReuse/ActiveRecord + + def self.prune(relation_name, relation, dry_run: true, root: nil) + root ||= '../repositories' + + known_paths = Set.new + listing(relation_name, relation) { |p| known_paths << "#{root}/#{p.repository.disk_path}" } + + marked_for_deletion = Set.new(Dir["#{root}/@hashed/*/*/*"]) + marked_for_deletion.reject! do |path| + base = path.gsub(/\.(\w+\.)?git$/, '') + known_paths.include?(base) + end + + if marked_for_deletion.empty? + $stdout.puts "No orphaned directories found. Nothing to do!" + else + n = marked_for_deletion.size + $stdout.puts "Found #{n} orphaned #{'directory'.pluralize(n)}" + $stdout.puts "Dry run. (Run again with FORCE=1 to delete). We would have deleted:" if dry_run + end + + marked_for_deletion.each do |p| + p = Pathname.new(p) + if dry_run + $stdout.puts " - #{p}" + else + $stdout.puts "Removing #{p}" + p.rmtree + end + end + end end end end diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index c09d8170d17..f0b08bb6b6a 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -4,7 +4,7 @@ module Gitlab module Jira # Gitlab JIRA HTTP client to be used with jira-ruby gem, this subclasses JIRA::HTTPClient. # Uses Gitlab::HTTP to make requests to JIRA REST API. - # The parent class implementation can be found at: https://github.com/sumoheavy/jira-ruby/blob/v1.7.0/lib/jira/http_client.rb + # The parent class implementation can be found at: https://github.com/sumoheavy/jira-ruby/blob/master/lib/jira/http_client.rb class HttpClient < JIRA::HttpClient extend ::Gitlab::Utils::Override @@ -43,6 +43,8 @@ module Gitlab result end + private + def auth_params return {} unless @options[:username] && @options[:password] @@ -54,8 +56,6 @@ module Gitlab } end - private - def get_cookies cookie_array = @cookies.values.map { |cookie| "#{cookie.name}=#{cookie.value[0]}" } cookie_array += Array(@options[:additional_cookies]) if @options.key?(:additional_cookies) diff --git a/lib/gitlab/kubernetes/cilium_network_policy.rb b/lib/gitlab/kubernetes/cilium_network_policy.rb index 9043932bbe5..f77b3e8de99 100644 --- a/lib/gitlab/kubernetes/cilium_network_policy.rb +++ b/lib/gitlab/kubernetes/cilium_network_policy.rb @@ -12,7 +12,7 @@ module Gitlab # We are modeling existing kubernetes resource and don't have # control over amount of parameters. # rubocop:disable Metrics/ParameterLists - def initialize(name:, namespace:, selector:, ingress:, resource_version: nil, description: nil, labels: nil, creation_timestamp: nil, egress: nil) + def initialize(name:, namespace:, selector:, ingress:, resource_version: nil, description: nil, labels: nil, creation_timestamp: nil, egress: nil, annotations: nil) @name = name @description = description @namespace = namespace @@ -22,6 +22,7 @@ module Gitlab @resource_version = resource_version @ingress = ingress @egress = egress + @annotations = annotations end # rubocop:enable Metrics/ParameterLists @@ -37,6 +38,7 @@ module Gitlab name: metadata[:name], description: policy[:description], namespace: metadata[:namespace], + annotations: metadata[:annotations], resource_version: metadata[:resourceVersion], labels: metadata[:labels], selector: spec[:endpointSelector], @@ -57,6 +59,7 @@ module Gitlab name: metadata[:name], description: resource[:description], namespace: metadata[:namespace], + annotations: metadata[:annotations]&.to_h, resource_version: metadata[:resourceVersion], labels: metadata[:labels]&.to_h, creation_timestamp: metadata[:creationTimestamp], @@ -80,7 +83,7 @@ module Gitlab private - attr_reader :name, :description, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress + attr_reader :name, :description, :namespace, :labels, :creation_timestamp, :resource_version, :ingress, :egress, :annotations def selector @selector ||= {} @@ -90,6 +93,7 @@ module Gitlab meta = { name: name, namespace: namespace } meta[:labels] = labels if labels meta[:resourceVersion] = resource_version if resource_version + meta[:annotations] = annotations if annotations meta end diff --git a/lib/gitlab/kubernetes/kubectl_cmd.rb b/lib/gitlab/kubernetes/kubectl_cmd.rb index e8fde28b44d..f3ac19e210a 100644 --- a/lib/gitlab/kubernetes/kubectl_cmd.rb +++ b/lib/gitlab/kubernetes/kubectl_cmd.rb @@ -17,7 +17,7 @@ module Gitlab def delete_crds_from_group(group) api_resources_args = %w(-o name --api-group).push(group) - api_resources(*api_resources_args) + " | xargs " + delete('--ignore-not-found', 'crd') + PodCmd.retry_command(api_resources(*api_resources_args) + " | xargs -r " + delete('--ignore-not-found', 'crd')) end def api_resources(*args) diff --git a/lib/gitlab/kubernetes/pod_cmd.rb b/lib/gitlab/kubernetes/pod_cmd.rb new file mode 100644 index 00000000000..e4c25424e69 --- /dev/null +++ b/lib/gitlab/kubernetes/pod_cmd.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + # Miscellaneous commands that run in the helm-install-image pod, tuned to + # the idiosynchrasies of the default shell of helm-install-image + module PodCmd + class << self + def retry_command(command, times: 3) + "for i in $(seq 1 #{times.to_i}); do #{command} && s=0 && break || s=$?; sleep 1s; echo \"Retrying ($i)...\"; done; (exit $s)" + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/action_cable_sampler.rb b/lib/gitlab/metrics/samplers/action_cable_sampler.rb index 9f4979fa673..043d2ae84cc 100644 --- a/lib/gitlab/metrics/samplers/action_cable_sampler.rb +++ b/lib/gitlab/metrics/samplers/action_cable_sampler.rb @@ -4,9 +4,9 @@ module Gitlab module Metrics module Samplers class ActionCableSampler < BaseSampler - SAMPLING_INTERVAL_SECONDS = 5 + DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 - def initialize(interval = SAMPLING_INTERVAL_SECONDS, action_cable: ::ActionCable.server) + def initialize(interval = nil, action_cable: ::ActionCable.server) super(interval) @action_cable = action_cable end diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index 39a49187e45..7f9055fed5d 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -9,7 +9,9 @@ module Gitlab attr_reader :interval # interval - The sampling interval in seconds. - def initialize(interval = self.class::SAMPLING_INTERVAL_SECONDS) + def initialize(interval = nil) + interval ||= ENV[interval_env_key]&.to_i + interval ||= self.class::DEFAULT_SAMPLING_INTERVAL_SECONDS interval_half = interval.to_f / 2 @interval = interval @@ -50,6 +52,14 @@ module Gitlab attr_reader :running + def sampler_class + self.class.name.demodulize + end + + def interval_env_key + "#{sampler_class.underscore.upcase}_INTERVAL_SECONDS" + end + def start_working @running = true diff --git a/lib/gitlab/metrics/samplers/database_sampler.rb b/lib/gitlab/metrics/samplers/database_sampler.rb index 9ee4b0960c5..60ae22df607 100644 --- a/lib/gitlab/metrics/samplers/database_sampler.rb +++ b/lib/gitlab/metrics/samplers/database_sampler.rb @@ -4,7 +4,7 @@ module Gitlab module Metrics module Samplers class DatabaseSampler < BaseSampler - SAMPLING_INTERVAL_SECONDS = 5 + DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 METRIC_PREFIX = 'gitlab_database_connection_pool_' diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index d295beb59f1..848a55e59ff 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -4,7 +4,7 @@ module Gitlab module Metrics module Samplers class PumaSampler < BaseSampler - SAMPLING_INTERVAL_SECONDS = 5 + DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 def metrics @metrics ||= init_metrics diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index dac9fbd1247..76175b465e4 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -6,7 +6,7 @@ module Gitlab module Metrics module Samplers class RubySampler < BaseSampler - SAMPLING_INTERVAL_SECONDS = 60 + DEFAULT_SAMPLING_INTERVAL_SECONDS = 60 GC_REPORT_BUCKETS = [0.005, 0.01, 0.02, 0.04, 0.07, 0.1, 0.5].freeze def initialize(*) diff --git a/lib/gitlab/metrics/samplers/threads_sampler.rb b/lib/gitlab/metrics/samplers/threads_sampler.rb index 05acef7ce0c..a460594fb59 100644 --- a/lib/gitlab/metrics/samplers/threads_sampler.rb +++ b/lib/gitlab/metrics/samplers/threads_sampler.rb @@ -4,7 +4,7 @@ module Gitlab module Metrics module Samplers class ThreadsSampler < BaseSampler - SAMPLING_INTERVAL_SECONDS = 5 + DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 KNOWN_PUMA_THREAD_NAMES = ['puma worker check pipe', 'puma server', 'puma threadpool reaper', 'puma threadpool trimmer', 'puma worker check pipe', 'puma stat payload'].freeze diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb index d7935d65e12..2fa324f3fea 100644 --- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb +++ b/lib/gitlab/metrics/samplers/unicorn_sampler.rb @@ -4,6 +4,8 @@ module Gitlab module Metrics module Samplers class UnicornSampler < BaseSampler + DEFAULT_SAMPLING_INTERVAL_SECONDS = 5 + def metrics @metrics ||= init_metrics end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index 43005303dec..9bbcd1e056c 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -17,6 +17,20 @@ module Gitlab RSS_PATTERN = /VmRSS:\s+(?<value>\d+)/.freeze MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/.freeze + def self.summary + proportional_mem = memory_usage_uss_pss + { + version: RUBY_DESCRIPTION, + gc_stat: GC.stat, + memory_rss: memory_usage_rss, + memory_uss: proportional_mem[:uss], + memory_pss: proportional_mem[:pss], + time_cputime: cpu_time, + time_realtime: real_time, + time_monotonic: monotonic_time + } + end + # Returns the current process' RSS (resident set size) in bytes. def self.memory_usage_rss sum_matches(PROC_STATUS_PATH, rss: RSS_PATTERN)[:rss].kilobytes diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index a6d8a778e05..79f1abe820f 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -41,7 +41,7 @@ module Gitlab end def with_open_files - @rewritten_fields.each do |field, tmp_path| + @rewritten_fields.keys.each do |field| raise "invalid field: #{field.inspect}" unless valid_field_name?(field) parsed_field = Rack::Utils.parse_nested_query(field) @@ -51,10 +51,10 @@ module Gitlab if value.nil? # we have a top level param, eg. field = 'foo' and not 'foo[bar]' raise "invalid field: #{field.inspect}" if field != key - value = open_file(@request.params, key, tmp_path.presence) + value = open_file(extract_upload_params_from(@request.params, with_prefix: key)) @open_files << value else - value = decorate_params_value(value, @request.params[key], tmp_path.presence) + value = decorate_params_value(value, @request.params[key]) end update_param(key, value) @@ -67,12 +67,12 @@ module Gitlab end # This function calls itself recursively - def decorate_params_value(path_hash, value_hash, path_override = nil) - unless path_hash.is_a?(Hash) && path_hash.count == 1 - raise "invalid path: #{path_hash.inspect}" + def decorate_params_value(hash_path, value_hash) + unless hash_path.is_a?(Hash) && hash_path.count == 1 + raise "invalid path: #{hash_path.inspect}" end - path_key, path_value = path_hash.first + path_key, path_value = hash_path.first unless value_hash.is_a?(Hash) && value_hash[path_key] raise "invalid value hash: #{value_hash.inspect}" @@ -80,19 +80,19 @@ module Gitlab case path_value when nil - value_hash[path_key] = open_file(value_hash.dig(path_key), '', path_override) + value_hash[path_key] = open_file(extract_upload_params_from(value_hash[path_key])) @open_files << value_hash[path_key] value_hash when Hash - decorate_params_value(path_value, value_hash[path_key], path_override) + decorate_params_value(path_value, value_hash[path_key]) value_hash else raise "unexpected path value: #{path_value.inspect}" end end - def open_file(params, key, path_override = nil) - ::UploadedFile.from_params(params, key, allowed_paths, path_override) + def open_file(params) + ::UploadedFile.from_params(params, allowed_paths) end # update_params ensures that both rails controllers and rack middleware can find @@ -111,6 +111,20 @@ module Gitlab private + def extract_upload_params_from(params, with_prefix: '') + param_key = "#{with_prefix}#{JWT_PARAM_SUFFIX}" + jwt_token = params[param_key] + raise "Empty JWT param: #{param_key}" if jwt_token.blank? + + payload = Gitlab::Workhorse.decode_jwt(jwt_token).first + raise "Invalid JWT payload: not a Hash" unless payload.is_a?(Hash) + + upload_params = payload.fetch(JWT_PARAM_FIXED_KEY, {}) + raise "Empty params for: #{param_key}" if upload_params.empty? + + upload_params + end + def valid_field_name?(name) # length validation return false if name.size >= REWRITTEN_FIELD_NAME_MAX_LENGTH @@ -149,82 +163,6 @@ module Gitlab end end - # TODO this class is meant to replace Handler when the feature flag - # upload_middleware_jwt_params_handler is removed - # See https://gitlab.com/gitlab-org/gitlab/-/issues/233895#roll-out-steps - class HandlerForJWTParams < Handler - def with_open_files - @rewritten_fields.keys.each do |field| - raise "invalid field: #{field.inspect}" unless valid_field_name?(field) - - parsed_field = Rack::Utils.parse_nested_query(field) - raise "unexpected field: #{field.inspect}" unless parsed_field.count == 1 - - key, value = parsed_field.first - if value.nil? # we have a top level param, eg. field = 'foo' and not 'foo[bar]' - raise "invalid field: #{field.inspect}" if field != key - - value = open_file(extract_upload_params_from(@request.params, with_prefix: key)) - @open_files << value - else - value = decorate_params_value(value, @request.params[key]) - end - - update_param(key, value) - end - - yield - ensure - @open_files.compact - .each(&:close) - end - - # This function calls itself recursively - def decorate_params_value(hash_path, value_hash) - unless hash_path.is_a?(Hash) && hash_path.count == 1 - raise "invalid path: #{hash_path.inspect}" - end - - path_key, path_value = hash_path.first - - unless value_hash.is_a?(Hash) && value_hash[path_key] - raise "invalid value hash: #{value_hash.inspect}" - end - - case path_value - when nil - value_hash[path_key] = open_file(extract_upload_params_from(value_hash[path_key])) - @open_files << value_hash[path_key] - value_hash - when Hash - decorate_params_value(path_value, value_hash[path_key]) - value_hash - else - raise "unexpected path value: #{path_value.inspect}" - end - end - - def open_file(params) - ::UploadedFile.from_params_without_field(params, allowed_paths) - end - - private - - def extract_upload_params_from(params, with_prefix: '') - param_key = "#{with_prefix}#{JWT_PARAM_SUFFIX}" - jwt_token = params[param_key] - raise "Empty JWT param: #{param_key}" if jwt_token.blank? - - payload = Gitlab::Workhorse.decode_jwt(jwt_token).first - raise "Invalid JWT payload: not a Hash" unless payload.is_a?(Hash) - - upload_params = payload.fetch(JWT_PARAM_FIXED_KEY, {}) - raise "Empty params for: #{param_key}" if upload_params.empty? - - upload_params - end - end - def initialize(app) @app = app end @@ -235,22 +173,12 @@ module Gitlab message = ::Gitlab::Workhorse.decode_jwt(encoded_message)[0] - handler_class.new(env, message).with_open_files do + ::Gitlab::Middleware::Multipart::Handler.new(env, message).with_open_files do @app.call(env) end rescue UploadedFile::InvalidPathError => e [400, { 'Content-Type' => 'text/plain' }, e.message] end - - private - - def handler_class - if Feature.enabled?(:upload_middleware_jwt_params_handler, default_enabled: true) - ::Gitlab::Middleware::Multipart::HandlerForJWTParams - else - ::Gitlab::Middleware::Multipart::Handler - end - end end end end diff --git a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb index bf8d4b202b6..133d777fc32 100644 --- a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb +++ b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb @@ -15,23 +15,39 @@ module Gitlab # schedules a job which parses peek profile data and adds them # to a structured log + # rubocop:disable Gitlab/ModuleWithInstanceVariables def enqueue_stats_job(request_id) return unless gather_stats? - @client.sadd(GitlabPerformanceBarStatsWorker::STATS_KEY, request_id) # rubocop:disable Gitlab/ModuleWithInstanceVariables + @client.sadd(GitlabPerformanceBarStatsWorker::STATS_KEY, request_id) return unless uuid = Gitlab::ExclusiveLease.new( GitlabPerformanceBarStatsWorker::LEASE_KEY, timeout: GitlabPerformanceBarStatsWorker::LEASE_TIMEOUT ).try_obtain - GitlabPerformanceBarStatsWorker.perform_in(GitlabPerformanceBarStatsWorker::WORKER_DELAY, uuid) + # stats key should be periodically processed and deleted by + # GitlabPerformanceBarStatsWorker but if it doesn't happen for + # some reason, we set expiration for the stats key to avoid + # keeping millions of request ids which would be already expired + # anyway + # rubocop:disable Gitlab/ModuleWithInstanceVariables + @client.expire( + GitlabPerformanceBarStatsWorker::STATS_KEY, + GitlabPerformanceBarStatsWorker::STATS_KEY_EXPIRE + ) + + GitlabPerformanceBarStatsWorker.perform_in( + GitlabPerformanceBarStatsWorker::WORKER_DELAY, + uuid + ) end + # rubocop:enable Gitlab/ModuleWithInstanceVariables def gather_stats? return unless Feature.enabled?(:performance_bar_stats) - Gitlab.com? || !Rails.env.production? + Gitlab.com? || Gitlab.staging? || !Rails.env.production? end end end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 6ba36fadfa3..56eeea6e746 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -63,7 +63,8 @@ module Gitlab ProjectTemplate.new('salesforcedx', 'SalesforceDX', _('A project boilerplate for Salesforce App development with Salesforce Developer tools'), 'https://gitlab.com/gitlab-org/project-templates/salesforcedx'), ProjectTemplate.new('serverless_framework', 'Serverless Framework/JS', _('A basic page and serverless function that uses AWS Lambda, AWS API Gateway, and GitLab Pages'), 'https://gitlab.com/gitlab-org/project-templates/serverless-framework', 'illustrations/logos/serverless_framework.svg'), ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'), - ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management') + ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'), + ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux') ].freeze end diff --git a/lib/gitlab/prometheus/internal.rb b/lib/gitlab/prometheus/internal.rb index c2f4035821e..fe06b97add6 100644 --- a/lib/gitlab/prometheus/internal.rb +++ b/lib/gitlab/prometheus/internal.rb @@ -4,43 +4,39 @@ module Gitlab module Prometheus class Internal def self.uri - return if listen_address.blank? + return if server_address.blank? - if listen_address.starts_with?('0.0.0.0:') + if server_address.starts_with?('0.0.0.0:') # 0.0.0.0:9090 - port = ':' + listen_address.split(':').second + port = ':' + server_address.split(':').second 'http://localhost' + port - elsif listen_address.starts_with?(':') + elsif server_address.starts_with?(':') # :9090 - 'http://localhost' + listen_address + 'http://localhost' + server_address - elsif listen_address.starts_with?('http') + elsif server_address.starts_with?('http') # https://localhost:9090 - listen_address + server_address else # localhost:9090 - 'http://' + listen_address + 'http://' + server_address end end def self.server_address - uri&.strip&.sub(/^http[s]?:\/\//, '') - end - - def self.listen_address - Gitlab.config.prometheus.listen_address.to_s if Gitlab.config.prometheus + Gitlab.config.prometheus.server_address.to_s if Gitlab.config.prometheus rescue Settingslogic::MissingSetting - Gitlab::AppLogger.error('Prometheus listen_address is not present in config/gitlab.yml') + Gitlab::AppLogger.error('Prometheus server_address is not present in config/gitlab.yml') nil end def self.prometheus_enabled? - Gitlab.config.prometheus.enable if Gitlab.config.prometheus + Gitlab.config.prometheus.enabled if Gitlab.config.prometheus rescue Settingslogic::MissingSetting - Gitlab::AppLogger.error('prometheus.enable is not present in config/gitlab.yml') + Gitlab::AppLogger.error('prometheus.enabled is not present in config/gitlab.yml') false end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 1822b0c8bd5..c162ee545c6 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -170,7 +170,8 @@ module Gitlab end types Issue condition do - !quick_action_target.confidential? && + quick_action_target.issue_type_supports?(:confidentiality) && + !quick_action_target.confidential? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", quick_action_target) end command :confidential do diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index 6607c73a5c3..4934c12a339 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -26,7 +26,7 @@ module Gitlab end types Issue, MergeRequest condition do - current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + quick_action_target.supports_assignee? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end parse_params do |assignee_param| extract_users(assignee_param) diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 1986b7a1789..b56fd8278a1 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -9,53 +9,72 @@ module Gitlab included do # MergeRequest only quick actions definitions desc do - if Feature.enabled?(:merge_orchestration_service, quick_action_target.project, default_enabled: true) - if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) - _("Merge automatically (%{strategy})") % { strategy: preferred_strategy.humanize } - else - _("Merge immediately") - end + if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) + _("Merge automatically (%{strategy})") % { strategy: preferred_strategy.humanize } else - _('Merge (when the pipeline succeeds)') + _("Merge immediately") end end explanation do - if Feature.enabled?(:merge_orchestration_service, quick_action_target.project, default_enabled: true) - if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) - _("Schedules to merge this merge request (%{strategy}).") % { strategy: preferred_strategy.humanize } - else - _('Merges this merge request immediately.') - end + if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) + _("Schedules to merge this merge request (%{strategy}).") % { strategy: preferred_strategy.humanize } else - _('Merges this merge request when the pipeline succeeds.') + _('Merges this merge request immediately.') end end execution_message do - if Feature.enabled?(:merge_orchestration_service, quick_action_target.project, default_enabled: true) - if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) - _("Scheduled to merge this merge request (%{strategy}).") % { strategy: preferred_strategy.humanize } - else - _('Merged this merge request.') - end + if preferred_strategy = preferred_auto_merge_strategy(quick_action_target) + _("Scheduled to merge this merge request (%{strategy}).") % { strategy: preferred_strategy.humanize } else - _('Scheduled to merge this merge request when the pipeline succeeds.') + _('Merged this merge request.') end end types MergeRequest condition do - if Feature.enabled?(:merge_orchestration_service, quick_action_target.project, default_enabled: true) - quick_action_target.persisted? && - merge_orchestration_service.can_merge?(quick_action_target) - else - last_diff_sha = params && params[:merge_request_diff_head_sha] - quick_action_target.persisted? && - quick_action_target.mergeable_with_quick_action?(current_user, autocomplete_precheck: !last_diff_sha, last_diff_sha: last_diff_sha) - end + quick_action_target.persisted? && + merge_orchestration_service.can_merge?(quick_action_target) end command :merge do @updates[:merge] = params[:merge_request_diff_head_sha] end + types MergeRequest + desc do + _('Rebase source branch') + end + explanation do + _('Rebase source branch on the target branch.') + end + condition do + merge_request = quick_action_target + + next false unless merge_request.open? + next false unless merge_request.source_branch_exists? + + access_check = ::Gitlab::UserAccess + .new(current_user, container: merge_request.source_project) + + access_check.can_push_to_branch?(merge_request.source_branch) + end + command :rebase do + if quick_action_target.cannot_be_merged? + @execution_message[:rebase] = _('This merge request cannot be rebased while there are conflicts.') + next + end + + if quick_action_target.rebase_in_progress? + @execution_message[:rebase] = _('A rebase is already in progress.') + next + end + + # This will be used to avoid simultaneous "/merge" and "/rebase" actions + @updates[:rebase] = true + + branch = quick_action_target.source_branch + + @execution_message[:rebase] = _('Scheduled a rebase of branch %{branch}.') % { branch: branch } + end + desc 'Toggle the Draft status' explanation do noun = quick_action_target.to_ability_name.humanize(capitalize: false) @@ -135,6 +154,112 @@ module Gitlab @execution_message[:approve] = _('Approved the current merge request.') end + + desc do + if quick_action_target.allows_multiple_reviewers? + _('Assign reviewer(s)') + else + _('Assign reviewer') + end + end + explanation do |users| + reviewers = reviewers_to_add(users) + _('Assigns %{reviewer_users_sentence} as %{reviewer_text}.') % { reviewer_users_sentence: reviewer_users_sentence(users), + reviewer_text: 'reviewer'.pluralize(reviewers.size) } + end + execution_message do |users = nil| + reviewers = reviewers_to_add(users) + if reviewers.blank? + _("Failed to assign a reviewer because no user was found.") + else + _('Assigned %{reviewer_users_sentence} as %{reviewer_text}.') % { reviewer_users_sentence: reviewer_users_sentence(users), + reviewer_text: 'reviewer'.pluralize(reviewers.size) } + end + end + params do + quick_action_target.allows_multiple_reviewers? ? '@user1 @user2' : '@user' + end + types MergeRequest + condition do + Feature.enabled?(:merge_request_reviewers, project, default_enabled: :yaml) && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + parse_params do |reviewer_param| + extract_users(reviewer_param) + end + command :assign_reviewer, :reviewer do |users| + next if users.empty? + + if quick_action_target.allows_multiple_reviewers? + @updates[:reviewer_ids] ||= quick_action_target.reviewers.map(&:id) + @updates[:reviewer_ids] |= users.map(&:id) + else + @updates[:reviewer_ids] = [users.first.id] + end + end + + desc do + if quick_action_target.allows_multiple_reviewers? + _('Remove all or specific reviewer(s)') + else + _('Remove reviewer') + end + end + explanation do |users = nil| + reviewers = reviewers_for_removal(users) + _("Removes %{reviewer_text} %{reviewer_references}.") % + { reviewer_text: 'reviewer'.pluralize(reviewers.size), reviewer_references: reviewers.map(&:to_reference).to_sentence } + end + execution_message do |users = nil| + reviewers = reviewers_for_removal(users) + _("Removed %{reviewer_text} %{reviewer_references}.") % + { reviewer_text: 'reviewer'.pluralize(reviewers.size), reviewer_references: reviewers.map(&:to_reference).to_sentence } + end + params do + quick_action_target.allows_multiple_reviewers? ? '@user1 @user2' : '' + end + types MergeRequest + condition do + quick_action_target.persisted? && + Feature.enabled?(:merge_request_reviewers, project, default_enabled: :yaml) && + quick_action_target.reviewers.any? && + current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) + end + parse_params do |unassign_reviewer_param| + # When multiple users are assigned, all will be unassigned if multiple reviewers are no longer allowed + extract_users(unassign_reviewer_param) if quick_action_target.allows_multiple_reviewers? + end + command :unassign_reviewer, :remove_reviewer do |users = nil| + if quick_action_target.allows_multiple_reviewers? && users&.any? + @updates[:reviewer_ids] ||= quick_action_target.reviewers.map(&:id) + @updates[:reviewer_ids] -= users.map(&:id) + else + @updates[:reviewer_ids] = [] + end + end + end + + def reviewer_users_sentence(users) + reviewers_to_add(users).map(&:to_reference).to_sentence + end + + def reviewers_for_removal(users) + reviewers = quick_action_target.reviewers + if users.present? && quick_action_target.allows_multiple_reviewers? + users + else + reviewers + end + end + + def reviewers_to_add(users) + return if users.blank? + + if quick_action_target.allows_multiple_reviewers? + users + else + [users.first] + end end def merge_orchestration_service diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index 7c336153e32..2a94fb91880 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -10,14 +10,70 @@ module Gitlab def self.configure(rack_attack) # This adds some methods used by our throttles to the `Rack::Request` rack_attack::Request.include(Gitlab::RackAttack::Request) - # Send the Retry-After header so clients (e.g. python-gitlab) can make good choices about delays - Rack::Attack.throttled_response_retry_after_header = true + + # This is Rack::Attack::DEFAULT_THROTTLED_RESPONSE, modified to allow a custom response + Rack::Attack.throttled_response = lambda do |env| + throttled_headers = Gitlab::RackAttack.throttled_response_headers( + env['rack.attack.matched'], env['rack.attack.match_data'] + ) + [429, { 'Content-Type' => 'text/plain' }.merge(throttled_headers), [Gitlab::Throttle.rate_limiting_response_text]] + end + # Configure the throttles configure_throttles(rack_attack) configure_user_allowlist end + # Rate Limit HTTP headers are not standardized anywhere. This is the latest + # draft submitted to IETF: + # https://github.com/ietf-wg-httpapi/ratelimit-headers/blob/main/draft-ietf-httpapi-ratelimit-headers.md + # + # This method implement the most viable parts of the headers. Those headers + # will be sent back to the client when it gets throttled. + # + # - RateLimit-Limit: indicates the request quota associated to the client + # in 60 seconds. The time window for the quota here is supposed to be + # mirrored to throttle_*_period_in_seconds application settings. However, + # our HAProxy as well as some ecosystem libraries are using a fixed + # 60-second window. Therefore, the returned limit is approximately rounded + # up to fit into that window. + # + # - RateLimit-Observed: indicates the current request amount associated to + # the client within the time window. + # + # - RateLimit-Remaining: indicates the remaining quota within the time + # window. It is the result of RateLimit-Limit - RateLimit-Remaining + # + # - Retry-After: the remaining duration in seconds until the quota is + # reset. This is a standardized HTTP header: + # https://tools.ietf.org/html/rfc7231#page-69 + # + # - RateLimit-Reset: the point of time that the request quota is reset, in Unix time + # + # - RateLimit-ResetTime: the point of time that the request quota is reset, in HTTP date format + def self.throttled_response_headers(matched, match_data) + # Match data example: + # {:discriminator=>"127.0.0.1", :count=>12, :period=>60 seconds, :limit=>1, :epoch_time=>1609833930} + # Source: https://github.com/rack/rack-attack/blob/v6.3.0/lib/rack/attack/throttle.rb#L33 + period = match_data[:period] + limit = match_data[:limit] + rounded_limit = (limit.to_f * 1.minute / match_data[:period]).ceil + observed = match_data[:count] + now = match_data[:epoch_time] + retry_after = period - (now % period) + reset_time = Time.at(now + retry_after) # rubocop:disable Rails/TimeZone + { + 'RateLimit-Name' => matched.to_s, + 'RateLimit-Limit' => rounded_limit.to_s, + 'RateLimit-Observed' => observed.to_s, + 'RateLimit-Remaining' => (limit > observed ? limit - observed : 0).to_s, + 'RateLimit-Reset' => reset_time.to_i.to_s, + 'RateLimit-ResetTime' => reset_time.httpdate, + 'Retry-After' => retry_after.to_s + } + end + def self.configure_user_allowlist @user_allowlist = nil user_allowlist diff --git a/lib/gitlab/sourcegraph.rb b/lib/gitlab/sourcegraph.rb index 231d5aea129..7ef6ab32bd4 100644 --- a/lib/gitlab/sourcegraph.rb +++ b/lib/gitlab/sourcegraph.rb @@ -13,7 +13,8 @@ module Gitlab end def feature_enabled?(actor = nil) - feature.enabled?(actor) + # Some CI jobs grep for Feature.enabled? in our codebase, so it is important this reference stays around. + Feature.enabled?(:sourcegraph, actor) end private diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index e84937ec4ad..b659bff52ad 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -23,7 +23,12 @@ module Gitlab end def content - @finder.read(@path) + blob = @finder.read(@path) + [description, blob].compact.join("\n") + end + + def description + # override with a comment to be placed at the top of the blob. end # Present for compatibility with license templates, which can replace text diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb index 3b516bb862a..09643cfb619 100644 --- a/lib/gitlab/template/dockerfile_template.rb +++ b/lib/gitlab/template/dockerfile_template.rb @@ -3,9 +3,8 @@ module Gitlab module Template class DockerfileTemplate < BaseTemplate - def content - explanation = "# This file is a template, and might need editing before it works on your project." - [explanation, super].join("\n") + def description + "# This file is a template, and might need editing before it works on your project." end class << self diff --git a/lib/gitlab/template/gitlab_ci_syntax_yml_template.rb b/lib/gitlab/template/gitlab_ci_syntax_yml_template.rb new file mode 100644 index 00000000000..3bf3a28d3c5 --- /dev/null +++ b/lib/gitlab/template/gitlab_ci_syntax_yml_template.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Template + class GitlabCiSyntaxYmlTemplate < BaseTemplate + class << self + def extension + '.gitlab-ci.yml' + end + + def categories + { + 'General' => '' + } + end + + def base_dir + Rails.root.join('lib/gitlab/ci/syntax_templates') + end + + def finder(project = nil) + Gitlab::Template::Finders::GlobalTemplateFinder.new( + self.base_dir, self.extension, self.categories + ) + end + end + end + end +end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index e12af6bf0a4..c295cc75da5 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -5,9 +5,8 @@ module Gitlab class GitlabCiYmlTemplate < BaseTemplate BASE_EXCLUDED_PATTERNS = [%r{\.latest\.}].freeze - def content - explanation = "# This file is a template, and might need editing before it works on your project." - [explanation, super].join("\n") + def description + "# This file is a template, and might need editing before it works on your project." end class << self diff --git a/lib/gitlab/template/metrics_dashboard_template.rb b/lib/gitlab/template/metrics_dashboard_template.rb index 88fc3007b63..469f97d7cb1 100644 --- a/lib/gitlab/template/metrics_dashboard_template.rb +++ b/lib/gitlab/template/metrics_dashboard_template.rb @@ -3,9 +3,8 @@ module Gitlab module Template class MetricsDashboardTemplate < BaseTemplate - def content - explanation = "# This file is a template, and might need editing before it works on your project." - [explanation, super].join("\n") + def description + "# This file is a template, and might need editing before it works on your project." end class << self diff --git a/lib/gitlab/throttle.rb b/lib/gitlab/throttle.rb index aebf8d92cb3..520075012e8 100644 --- a/lib/gitlab/throttle.rb +++ b/lib/gitlab/throttle.rb @@ -2,6 +2,8 @@ module Gitlab class Throttle + DEFAULT_RATE_LIMITING_RESPONSE_TEXT = 'Retry later' + def self.settings Gitlab::CurrentSettings.current_application_settings end @@ -46,5 +48,9 @@ module Gitlab { limit: limit_proc, period: period_proc } end + + def self.rate_limiting_response_text + (settings.rate_limiting_response_text.presence || DEFAULT_RATE_LIMITING_RESPONSE_TEXT) + "\n" + end end end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 618e359211b..ca4afb4c19c 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -24,7 +24,9 @@ module Gitlab Gitlab::CurrentSettings.snowplow_enabled? end - def event(category, action, label: nil, property: nil, value: nil, context: nil) + def event(category, action, label: nil, property: nil, value: nil, context: [], standard_context: nil) + context.push(standard_context.to_context) if standard_context + snowplow.event(category, action, label: label, property: property, value: value, context: context) product_analytics.event(category, action, label: label, property: property, value: value, context: context) end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb new file mode 100644 index 00000000000..71dfe27dd5a --- /dev/null +++ b/lib/gitlab/tracking/standard_context.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + class StandardContext + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-1'.freeze + + def initialize(namespace: nil, project: nil, **data) + @namespace = namespace + @project = project + @data = data + end + + def namespace_id + namespace&.id + end + + def project_id + @project&.id + end + + def to_context + SnowplowTracker::SelfDescribingJson.new(GITLAB_STANDARD_SCHEMA_URL, to_h) + end + + private + + def namespace + @namespace || @project&.namespace + end + + def to_h + public_methods(false).each_with_object({}) do |method, hash| + next if method == :to_context + + hash[method] = public_send(method) # rubocop:disable GitlabSecurity/PublicSend + end.merge(@data) + end + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index ce59e10241e..f98c488bbe5 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -18,6 +18,8 @@ module Gitlab def build(object, **options) # Objects are sometimes wrapped in a BatchLoader instance case object.itself + when Board + board_url(object, **options) when ::Ci::Build instance.project_job_url(object.project, object, **options) when Commit @@ -52,6 +54,14 @@ module Gitlab end # rubocop:enable Metrics/CyclomaticComplexity + def board_url(board, **options) + if board.project_board? + instance.project_board_url(board.resource_parent, board, **options) + else + instance.group_board_url(board.resource_parent, board, **options) + end + end + def commit_url(commit, **options) return '' unless commit.project diff --git a/lib/gitlab/usage/metric.rb b/lib/gitlab/usage/metric.rb new file mode 100644 index 00000000000..e1648c78168 --- /dev/null +++ b/lib/gitlab/usage/metric.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + class Metric + include ActiveModel::Model + + InvalidMetricError = Class.new(RuntimeError) + + attr_accessor :default_generation_path, :value + + validates :default_generation_path, presence: true + + def definition + self.class.definitions[default_generation_path] + end + + def unflatten_default_path + unflatten(default_generation_path.split('.'), value) + end + + class << self + def definitions + @definitions ||= Gitlab::Usage::MetricDefinition.definitions + end + + def dictionary + definitions.map { |key, definition| definition.to_dictionary } + end + end + + private + + def unflatten(keys, value) + loop do + value = { keys.pop.to_sym => value } + break if keys.blank? + end + value + end + end + end +end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb new file mode 100644 index 00000000000..96e572bb3db --- /dev/null +++ b/lib/gitlab/usage/metric_definition.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + class MetricDefinition + METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') + + attr_reader :path + attr_reader :attributes + + def initialize(path, opts = {}) + @path = path + @attributes = opts + end + + # The key is defined by default_generation and full_path + def key + full_path[default_generation.to_sym] + end + + def to_h + attributes + end + + def validate! + self.class.schemer.validate(attributes.stringify_keys).map do |error| + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("#{error["details"] || error['data_pointer']} for `#{path}`")) + end + end + + alias_method :to_dictionary, :to_h + + class << self + def paths + @paths ||= [Rails.root.join('config', 'metrics', '**', '*.yml')] + end + + def definitions + @definitions ||= load_all! + end + + def schemer + @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH)) + end + + private + + def load_all! + paths.each_with_object({}) do |glob_path, definitions| + load_all_from_path!(definitions, glob_path) + end + end + + def load_from_file(path) + definition = File.read(path) + definition = YAML.safe_load(definition) + definition.deep_symbolize_keys! + + self.new(path, definition).tap(&:validate!) + rescue => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new(e.message)) + end + + def load_all_from_path!(definitions, glob_path) + Dir.glob(glob_path).each do |path| + definition = load_from_file(path) + + if previous = definitions[definition.key] + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(Metric::InvalidMetricError.new("Metric '#{definition.key}' is already defined in '#{previous.path}'")) + end + + definitions[definition.key] = definition + end + end + end + + private + + def method_missing(method, *args) + attributes[method] || super + end + end + end +end + +Gitlab::Usage::MetricDefinition.prepend_if_ee('EE::Gitlab::Usage::MetricDefinition') diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb index ca7699e64e1..ed9dad37f3e 100644 --- a/lib/gitlab/usage_data_counters.rb +++ b/lib/gitlab/usage_data_counters.rb @@ -3,7 +3,7 @@ module Gitlab module UsageDataCounters COUNTERS = [ - GuestPackageEventCounter, + PackageEventCounter, WikiPageCounter, WebIdeCounter, NoteCounter, diff --git a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml index b7c0abae227..4966afd534a 100644 --- a/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml +++ b/lib/gitlab/usage_data_counters/aggregated_metrics/common.yml @@ -8,6 +8,9 @@ # Corresponding feature flag should have `default_enabled` attribute set to `false`. # This attribute is OPTIONAL and can be omitted, when `feature_flag` is missing no feature flag will be checked. --- +- name: compliance_features_track_unique_visits_union + operator: OR + events: ['g_compliance_audit_events', 'g_compliance_dashboard', 'i_compliance_audit_events', 'a_compliance_audit_events_api', 'i_compliance_credential_inventory'] - name: product_analytics_test_metrics_union operator: OR events: ['i_search_total', 'i_search_advanced', 'i_search_paid'] @@ -22,7 +25,6 @@ 'incident_management_alert_todo', 'incident_management_alert_create_incident' ] - feature_flag: usage_data_incident_management_alerts_total_unique_counts - name: incident_management_incidents_total_unique_counts operator: OR events: [ @@ -38,4 +40,3 @@ 'incident_management_incident_unrelate', 'incident_management_incident_change_confidential' ] - feature_flag: usage_data_incident_management_incidents_total_unique_counts diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb new file mode 100644 index 00000000000..572ad866895 --- /dev/null +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab::UsageDataCounters + class CiTemplateUniqueCounter + REDIS_SLOT = 'ci_templates'.freeze + + TEMPLATE_TO_EVENT = { + 'Auto-DevOps.gitlab-ci.yml' => 'auto_devops', + 'AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml' => 'aws_cf_deploy_ec2', + 'AWS/Deploy-ECS.gitlab-ci.yml' => 'aws_deploy_ecs', + 'Jobs/Build.gitlab-ci.yml' => 'auto_devops_build', + 'Jobs/Deploy.gitlab-ci.yml' => 'auto_devops_deploy', + 'Jobs/Deploy.latest.gitlab-ci.yml' => 'auto_devops_deploy_latest', + 'Security/SAST.gitlab-ci.yml' => 'security_sast', + 'Security/Secret-Detection.gitlab-ci.yml' => 'security_secret_detection', + 'Terraform/Base.latest.gitlab-ci.yml' => 'terraform_base_latest' + }.freeze + + class << self + def track_unique_project_event(project_id:, template:) + return if Feature.disabled?(:usage_data_track_ci_templates_unique_projects, default_enabled: :yaml) + + if event = unique_project_event(template) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event, values: project_id) + end + end + + private + + def unique_project_event(template) + if name = TEMPLATE_TO_EVENT[template] + "p_#{REDIS_SLOT}_#{name}" + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/counter_events/guest_package_events.yml b/lib/gitlab/usage_data_counters/counter_events/guest_package_events.yml deleted file mode 100644 index a9b9f8ea235..00000000000 --- a/lib/gitlab/usage_data_counters/counter_events/guest_package_events.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -- i_package_composer_guest_delete -- i_package_composer_guest_pull -- i_package_composer_guest_push -- i_package_conan_guest_delete -- i_package_conan_guest_pull -- i_package_conan_guest_push -- i_package_container_guest_delete -- i_package_container_guest_pull -- i_package_container_guest_push -- i_package_debian_guest_delete -- i_package_debian_guest_pull -- i_package_debian_guest_push -- i_package_generic_guest_delete -- i_package_generic_guest_pull -- i_package_generic_guest_push -- i_package_golang_guest_delete -- i_package_golang_guest_pull -- i_package_golang_guest_push -- i_package_maven_guest_delete -- i_package_maven_guest_pull -- i_package_maven_guest_push -- i_package_npm_guest_delete -- i_package_npm_guest_pull -- i_package_npm_guest_push -- i_package_nuget_guest_delete -- i_package_nuget_guest_pull -- i_package_nuget_guest_push -- i_package_pypi_guest_delete -- i_package_pypi_guest_pull -- i_package_pypi_guest_push -- i_package_tag_guest_delete -- i_package_tag_guest_pull -- i_package_tag_guest_push diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml new file mode 100644 index 00000000000..f6bddabdd44 --- /dev/null +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -0,0 +1,46 @@ +--- +- i_package_composer_delete_package +- i_package_composer_pull_package +- i_package_composer_push_package +- i_package_conan_delete_package +- i_package_conan_pull_package +- i_package_conan_push_package +- i_package_container_delete_package +- i_package_container_pull_package +- i_package_container_push_package +- i_package_debian_delete_package +- i_package_debian_pull_package +- i_package_debian_push_package +- i_package_delete_package +- i_package_delete_package_by_deploy_token +- i_package_delete_package_by_guest +- i_package_delete_package_by_user +- i_package_generic_delete_package +- i_package_generic_pull_package +- i_package_generic_push_package +- i_package_golang_delete_package +- i_package_golang_pull_package +- i_package_golang_push_package +- i_package_maven_delete_package +- i_package_maven_pull_package +- i_package_maven_push_package +- i_package_npm_delete_package +- i_package_npm_pull_package +- i_package_npm_push_package +- i_package_nuget_delete_package +- i_package_nuget_pull_package +- i_package_nuget_push_package +- i_package_pull_package +- i_package_pull_package_by_deploy_token +- i_package_pull_package_by_guest +- i_package_pull_package_by_user +- i_package_push_package +- i_package_push_package_by_deploy_token +- i_package_push_package_by_guest +- i_package_push_package_by_user +- i_package_pypi_delete_package +- i_package_pypi_pull_package +- i_package_pypi_push_package +- i_package_tag_delete_package +- i_package_tag_pull_package +- i_package_tag_push_package diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index eeb26c11bfa..bef3fc7b504 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -53,7 +53,7 @@ module Gitlab return unless Feature.enabled?(:track_editor_edit_actions, default_enabled: true) return unless author - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author.id, action, time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) end def count_unique(actions, date_from, date_to) diff --git a/lib/gitlab/usage_data_counters/guest_package_event_counter.rb b/lib/gitlab/usage_data_counters/guest_package_event_counter.rb deleted file mode 100644 index a9bcbfadda2..00000000000 --- a/lib/gitlab/usage_data_counters/guest_package_event_counter.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module UsageDataCounters - class GuestPackageEventCounter < BaseCounter - KNOWN_EVENTS_PATH = File.expand_path('counter_events/guest_package_events.yml', __dir__) - KNOWN_EVENTS = YAML.safe_load(File.read(KNOWN_EVENTS_PATH)).freeze - PREFIX = 'package_guest' - end - end -end diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index b61720c7638..47361d831b2 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -39,20 +39,31 @@ module Gitlab # # Usage: # - # * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event(user_id, 'g_compliance_dashboard') + # * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event('g_compliance_dashboard', values: user_id) # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current) class << self include Gitlab::Utils::UsageData - def track_event(value, event_name, time = Time.zone.now) - track(value, event_name, time: time) - end - - def track_event_in_context(value, event_name, context, time = Time.zone.now) + # Track unique events + # + # event_name - The event name. + # values - One or multiple values counted. + # time - Time of the action, set to Time.current. + def track_event(event_name, values:, time: Time.current) + track(values, event_name, time: time) + end + + # Track unique events + # + # event_name - The event name. + # values - One or multiple values counted. + # context - Event context, plan level tracking. + # time - Time of the action, set to Time.current. + def track_event_in_context(event_name, values:, context:, time: Time.zone.now) return if context.blank? return unless context.in?(valid_context_list) - track(value, event_name, context: context, time: time) + track(values, event_name, context: context, time: time) end def unique_events(event_names:, start_date:, end_date:, context: '') @@ -114,16 +125,16 @@ module Gitlab private - def track(value, event_name, context: '', time: Time.zone.now) + def track(values, event_name, context: '', time: Time.zone.now) return unless Gitlab::CurrentSettings.usage_ping_enabled? event = event_for(event_name) raise UnknownEvent, "Unknown event #{event_name}" unless event.present? - Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: value, expiry: expiry(event)) + Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) end - # The aray of valid context on which we allow tracking + # The array of valid context on which we allow tracking def valid_context_list Plan.all_plans end diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 0fed8e1c211..f649e7f407d 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -148,7 +148,7 @@ module Gitlab return unless Feature.enabled?(:track_issue_activity_actions, default_enabled: true) return unless author - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author.id, action, time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id, time: time) end end end diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 25cf388aedf..4cbde0c0372 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -248,6 +248,26 @@ redis_slot: testing aggregation: weekly feature_flag: usage_data_i_testing_test_case_parsed +- name: i_testing_metrics_report_widget_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_metrics_report_widget_total +- name: i_testing_group_code_coverage_visit_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_group_code_coverage_visit_total +- name: i_testing_full_code_quality_report_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_full_code_quality_report_total +- name: i_testing_web_performance_widget_total + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_i_testing_web_performance_widget_total # Project Management group - name: g_project_management_issue_title_changed category: issues_edit @@ -425,3 +445,126 @@ redis_slot: snippets aggregation: weekly feature_flag: usage_data_i_snippets_show +# Merge request counters +- name: i_code_review_mr_diffs + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_mr_diffs +- name: i_code_review_user_single_file_diffs + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_single_file_diffs +- name: i_code_review_mr_single_file_diffs + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_mr_single_file_diffs +- name: i_code_review_user_create_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_mr +- name: i_code_review_user_close_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_close_mr +- name: i_code_review_user_reopen_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_reopen_mr +- name: i_code_review_user_merge_mr + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_merge_mr +- name: i_code_review_user_create_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_mr_comment +- name: i_code_review_user_edit_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_edit_mr_comment +- name: i_code_review_user_remove_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_remove_mr_comment +- name: i_code_review_user_create_review_note + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_review_note +- name: i_code_review_user_publish_review + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_publish_review +- name: i_code_review_user_create_multiline_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_create_multiline_mr_comment +- name: i_code_review_user_edit_multiline_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_edit_multiline_mr_comment +- name: i_code_review_user_remove_multiline_mr_comment + redis_slot: code_review + category: code_review + aggregation: weekly + feature_flag: usage_data_i_code_review_user_remove_multiline_mr_comment +# Terraform +- name: p_terraform_state_api_unique_users + category: terraform + redis_slot: terraform + aggregation: weekly + feature_flag: usage_data_p_terraform_state_api_unique_users +# CI templates +- name: p_ci_templates_auto_devops + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_aws_cf_deploy_ec2 + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_auto_devops_build + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_auto_devops_deploy + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_auto_devops_deploy_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_security_sast + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_security_secret_detection + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects +- name: p_ci_templates_terraform_base_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly + feature_flag: usage_data_track_ci_templates_unique_projects diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index 4c3138dc000..78a2a587b34 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -1,331 +1,111 @@ --- -- name: i_package_composer_deploy_token_delete - category: composer_packages +- name: i_package_composer_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_composer_deploy_token_pull - category: composer_packages +- name: i_package_composer_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_composer_deploy_token_push - category: composer_packages +- name: i_package_conan_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_composer_user_delete - category: composer_packages +- name: i_package_conan_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_composer_user_pull - category: composer_packages +- name: i_package_container_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_composer_user_push - category: composer_packages +- name: i_package_container_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_conan_deploy_token_delete - category: conan_packages +- name: i_package_debian_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_conan_deploy_token_pull - category: conan_packages +- name: i_package_debian_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_conan_deploy_token_push - category: conan_packages +- name: i_package_generic_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_conan_user_delete - category: conan_packages +- name: i_package_generic_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_conan_user_pull - category: conan_packages +- name: i_package_golang_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_conan_user_push - category: conan_packages +- name: i_package_golang_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_container_deploy_token_delete - category: container_packages +- name: i_package_maven_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_container_deploy_token_pull - category: container_packages +- name: i_package_maven_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_container_deploy_token_push - category: container_packages +- name: i_package_npm_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_container_user_delete - category: container_packages +- name: i_package_npm_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_container_user_pull - category: container_packages +- name: i_package_nuget_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_container_user_push - category: container_packages +- name: i_package_nuget_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_debian_deploy_token_delete - category: debian_packages +- name: i_package_pypi_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_debian_deploy_token_pull - category: debian_packages +- name: i_package_pypi_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_debian_deploy_token_push - category: debian_packages +- name: i_package_tag_deploy_token + category: deploy_token_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis -- name: i_package_debian_user_delete - category: debian_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_debian_user_pull - category: debian_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_debian_user_push - category: debian_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_generic_deploy_token_delete - category: generic_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_generic_deploy_token_pull - category: generic_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_generic_deploy_token_push - category: generic_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_generic_user_delete - category: generic_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_generic_user_pull - category: generic_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_generic_user_push - category: generic_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_golang_deploy_token_delete - category: golang_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_golang_deploy_token_pull - category: golang_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_golang_deploy_token_push - category: golang_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_golang_user_delete - category: golang_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_golang_user_pull - category: golang_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_golang_user_push - category: golang_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_maven_deploy_token_delete - category: maven_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_maven_deploy_token_pull - category: maven_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_maven_deploy_token_push - category: maven_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_maven_user_delete - category: maven_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_maven_user_pull - category: maven_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_maven_user_push - category: maven_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_npm_deploy_token_delete - category: npm_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_npm_deploy_token_pull - category: npm_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_npm_deploy_token_push - category: npm_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_npm_user_delete - category: npm_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_npm_user_pull - category: npm_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_npm_user_push - category: npm_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_nuget_deploy_token_delete - category: nuget_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_nuget_deploy_token_pull - category: nuget_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_nuget_deploy_token_push - category: nuget_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_nuget_user_delete - category: nuget_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_nuget_user_pull - category: nuget_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_nuget_user_push - category: nuget_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_pypi_deploy_token_delete - category: pypi_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_pypi_deploy_token_pull - category: pypi_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_pypi_deploy_token_push - category: pypi_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_pypi_user_delete - category: pypi_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_pypi_user_pull - category: pypi_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_pypi_user_push - category: pypi_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_tag_deploy_token_delete - category: tag_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_tag_deploy_token_pull - category: tag_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_tag_deploy_token_push - category: tag_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_tag_user_delete - category: tag_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_tag_user_pull - category: tag_packages - aggregation: weekly - redis_slot: package - feature_flag: collect_package_events_redis -- name: i_package_tag_user_push - category: tag_packages +- name: i_package_tag_user + category: user_packages aggregation: weekly redis_slot: package feature_flag: collect_package_events_redis diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb new file mode 100644 index 00000000000..11d59257ed9 --- /dev/null +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + module MergeRequestActivityUniqueCounter + MR_DIFFS_ACTION = 'i_code_review_mr_diffs' + MR_DIFFS_SINGLE_FILE_ACTION = 'i_code_review_mr_single_file_diffs' + MR_DIFFS_USER_SINGLE_FILE_ACTION = 'i_code_review_user_single_file_diffs' + MR_CREATE_ACTION = 'i_code_review_user_create_mr' + MR_CLOSE_ACTION = 'i_code_review_user_close_mr' + MR_REOPEN_ACTION = 'i_code_review_user_reopen_mr' + MR_MERGE_ACTION = 'i_code_review_user_merge_mr' + MR_CREATE_COMMENT_ACTION = 'i_code_review_user_create_mr_comment' + MR_EDIT_COMMENT_ACTION = 'i_code_review_user_edit_mr_comment' + MR_REMOVE_COMMENT_ACTION = 'i_code_review_user_remove_mr_comment' + MR_CREATE_REVIEW_NOTE_ACTION = 'i_code_review_user_create_review_note' + MR_PUBLISH_REVIEW_ACTION = 'i_code_review_user_publish_review' + MR_CREATE_MULTILINE_COMMENT_ACTION = 'i_code_review_user_create_multiline_mr_comment' + MR_EDIT_MULTILINE_COMMENT_ACTION = 'i_code_review_user_edit_multiline_mr_comment' + MR_REMOVE_MULTILINE_COMMENT_ACTION = 'i_code_review_user_remove_multiline_mr_comment' + + class << self + def track_mr_diffs_action(merge_request:) + track_unique_action_by_merge_request(MR_DIFFS_ACTION, merge_request) + end + + def track_mr_diffs_single_file_action(merge_request:, user:) + track_unique_action_by_merge_request(MR_DIFFS_SINGLE_FILE_ACTION, merge_request) + track_unique_action_by_user(MR_DIFFS_USER_SINGLE_FILE_ACTION, user) + end + + def track_create_mr_action(user:) + track_unique_action_by_user(MR_CREATE_ACTION, user) + end + + def track_close_mr_action(user:) + track_unique_action_by_user(MR_CLOSE_ACTION, user) + end + + def track_merge_mr_action(user:) + track_unique_action_by_user(MR_MERGE_ACTION, user) + end + + def track_reopen_mr_action(user:) + track_unique_action_by_user(MR_REOPEN_ACTION, user) + end + + def track_create_comment_action(note:) + track_unique_action_by_user(MR_CREATE_COMMENT_ACTION, note.author) + track_multiline_unique_action(MR_CREATE_MULTILINE_COMMENT_ACTION, note) + end + + def track_edit_comment_action(note:) + track_unique_action_by_user(MR_EDIT_COMMENT_ACTION, note.author) + track_multiline_unique_action(MR_EDIT_MULTILINE_COMMENT_ACTION, note) + end + + def track_remove_comment_action(note:) + track_unique_action_by_user(MR_REMOVE_COMMENT_ACTION, note.author) + track_multiline_unique_action(MR_REMOVE_MULTILINE_COMMENT_ACTION, note) + end + + def track_create_review_note_action(user:) + track_unique_action_by_user(MR_CREATE_REVIEW_NOTE_ACTION, user) + end + + def track_publish_review_action(user:) + track_unique_action_by_user(MR_PUBLISH_REVIEW_ACTION, user) + end + + private + + def track_unique_action_by_merge_request(action, merge_request) + track_unique_action(action, merge_request.id) + end + + def track_unique_action_by_user(action, user) + return unless user + + track_unique_action(action, user.id) + end + + def track_unique_action(action, value) + Gitlab::UsageDataCounters::HLLRedisCounter.track_usage_event(action, value) + end + + def track_multiline_unique_action(action, note) + return unless note.is_a?(DiffNote) && note.multiline? + + track_unique_action_by_user(action, note.author) + end + end + end + end +end diff --git a/lib/gitlab/usage_data_counters/package_event_counter.rb b/lib/gitlab/usage_data_counters/package_event_counter.rb new file mode 100644 index 00000000000..700b518eae3 --- /dev/null +++ b/lib/gitlab/usage_data_counters/package_event_counter.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class PackageEventCounter < BaseCounter + KNOWN_EVENTS_PATH = File.expand_path('counter_events/package_events.yml', __dir__) + KNOWN_EVENTS = YAML.safe_load(File.read(KNOWN_EVENTS_PATH)).freeze + PREFIX = 'package_events' + end + end +end diff --git a/lib/gitlab/usage_data_counters/track_unique_events.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb index 95380ae0b1d..20da9665876 100644 --- a/lib/gitlab/usage_data_counters/track_unique_events.rb +++ b/lib/gitlab/usage_data_counters/track_unique_events.rb @@ -43,7 +43,7 @@ module Gitlab return unless Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(transformed_action.to_s) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, transformed_action.to_s, time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(transformed_action.to_s, values: author_id, time: time) track_git_write_action(author_id, transformed_action, time) end @@ -73,7 +73,7 @@ module Gitlab def track_git_write_action(author_id, transformed_action, time) return unless GIT_WRITE_ACTIONS.include?(transformed_action) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(author_id, GIT_WRITE_ACTION, time) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(GIT_WRITE_ACTION, values: author_id, time: time) end end end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index 3df54e74b4f..29f02a5912a 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -174,6 +174,18 @@ module Gitlab rescue IPAddr::InvalidAddressError end + # A safe alternative to String#downcase! + # + # This will make copies of frozen strings but downcase unfrozen + # strings in place, reducing allocations. + def safe_downcase!(str) + if str.frozen? + str.downcase + else + str.downcase! || str + end + end + # Converts a string to an Addressable::URI object. # If the string is not a valid URI, it returns nil. # Param uri_string should be a String object. diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 0d28a1cd035..baccadd9594 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -61,7 +61,10 @@ module Gitlab end def estimate_batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) - Gitlab::Database::PostgresHll::BatchDistinctCounter.new(relation, column).estimate_distinct_count(batch_size: batch_size, start: start, finish: finish) + Gitlab::Database::PostgresHll::BatchDistinctCounter + .new(relation, column) + .execute(batch_size: batch_size, start: start, finish: finish) + .estimated_distinct_count rescue ActiveRecord::StatementInvalid FALLBACK # catch all rescue should be removed as a part of feature flag rollout issue @@ -119,7 +122,7 @@ module Gitlab def track_usage_event(event_name, values) return unless Feature.enabled?(:"usage_data_#{event_name}", default_enabled: true) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(values, event_name.to_s) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values) end private @@ -142,7 +145,8 @@ module Gitlab def prometheus_server_address if Gitlab::Prometheus::Internal.prometheus_enabled? - Gitlab::Prometheus::Internal.server_address + # Stripping protocol from URI + Gitlab::Prometheus::Internal.uri&.strip&.sub(%r{^https?://}, '') elsif Gitlab::Consul::Internal.api_url Gitlab::Consul::Internal.discover_prometheus_server_address end diff --git a/lib/gitlab/uuid.rb b/lib/gitlab/uuid.rb index 12a4efabc44..80caf2c6788 100644 --- a/lib/gitlab/uuid.rb +++ b/lib/gitlab/uuid.rb @@ -9,6 +9,7 @@ module Gitlab production: "58dc0f06-936c-43b3-93bb-71693f1b6570" }.freeze + UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{4}\h{8}/.freeze NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze PACK_PATTERN = "NnnnnN".freeze @@ -17,6 +18,10 @@ module Gitlab Digest::UUID.uuid_v5(namespace_id, name) end + def v5?(string) + string.match(UUID_V5_PATTERN).present? + end + private def default_namespace_id diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index a22740ab9b7..76cf769d041 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -123,14 +123,6 @@ module Gitlab end end - def visibility_level_decreased? - return false unless visibility_level_previous_changes - - before, after = visibility_level_previous_changes - - before && after && after < before - end - def visibility_level_previous_changes previous_changes[:visibility_level] end diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb index 5873d9c2b99..9c967d99e3a 100644 --- a/lib/gitlab/webpack/manifest.rb +++ b/lib/gitlab/webpack/manifest.rb @@ -69,8 +69,8 @@ module Gitlab def manifest if Gitlab.config.webpack.dev_server.enabled - # Don't cache if we're in dev server mode, manifest may change ... - load_manifest + # Only cache at request level if we're in dev server mode, manifest may change ... + Gitlab::SafeRequestStore.fetch('manifest.json') { load_manifest } else # ... otherwise cache at class level, as JSON loading/parsing can be expensive strong_memoize(:manifest) { load_manifest } |