diff options
Diffstat (limited to 'lib/gitlab')
193 files changed, 2944 insertions, 1292 deletions
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 6afcd745d4e..d3c96a0f934 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -75,10 +75,10 @@ module Gitlab def protection_options { - "Not protected: Both developers and maintainers can push new commits, force push, or delete the branch." => PROTECTION_NONE, + "Not protected: Both developers and maintainers can push new commits and force push." => PROTECTION_NONE, "Protected against pushes: Developers cannot push new commits, but are allowed to accept merge requests to the branch. Maintainers can push to the branch." => PROTECTION_DEV_CAN_MERGE, - "Partially protected: Both developers and maintainers can push new commits, but cannot force push or delete the branch." => PROTECTION_DEV_CAN_PUSH, - "Fully protected: Developers cannot push new commits, but maintainers can. No-one can force push or delete the branch." => PROTECTION_FULL + "Partially protected: Both developers and maintainers can push new commits, but cannot force push." => PROTECTION_DEV_CAN_PUSH, + "Fully protected: Developers cannot push new commits, but maintainers can. No one can force push." => PROTECTION_FULL } end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb index 8e87245e62b..fda4ab0207d 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/code_stage_start.rb @@ -53,6 +53,10 @@ module Gitlab .on(mr_metrics_table[:merge_request_id].eq(mr_table[:id])) .join_sources end + + def include_in(query) + query.left_joins(merge_requests_closing_issues: { issue: [:metrics] }, metrics: []) + end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb index 4ca3c19051e..0cb081c64c4 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/issue_deployed_to_production.rb @@ -26,6 +26,10 @@ module Gitlab query.joins(merge_requests_closing_issues: { merge_request: [:metrics] }).where(mr_metrics_table[:first_deployed_to_production_at].gteq(mr_table[:created_at])) end # rubocop: enable CodeReuse/ActiveRecord + + def include_in(query) + query.left_joins(merge_requests_closing_issues: { merge_request: [:metrics] }) + end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb index fd30ab5277d..e191b0fe897 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/metrics_based_stage_event.rb @@ -20,6 +20,10 @@ module Gitlab def column_list [timestamp_projection] end + + def include_in(query) + super.left_joins(:metrics) + end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb index 8eb067ed0ec..945cecfcf8c 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/stage_event.rb @@ -61,6 +61,10 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord + def include_in(query) + query + end + def self.label_based? false end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 760f1352256..aa33f56582b 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -124,7 +124,10 @@ module Gitlab strong_memoize(:runner_project) do next unless runner&.project_type? - projects = runner.projects.take(2) # rubocop: disable CodeReuse/ActiveRecord + projects = ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/342147') do + runner.projects.take(2) # rubocop: disable CodeReuse/ActiveRecord + end + projects.first if projects.one? end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index f91a56a0cd2..7c37f67b766 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -11,6 +11,23 @@ module Gitlab # redirect_to(edit_project_path(@project), status: :too_many_requests) # end class ApplicationRateLimiter + def initialize(key, **options) + @key = key + @options = options + end + + def throttled? + self.class.throttled?(key, **options) + end + + def threshold_value + options[:threshold] || self.class.threshold(key) + end + + def interval_value + self.class.interval(key) + end + class << self # Application rate limits # @@ -73,7 +90,7 @@ module Gitlab value = 0 interval_value = interval || interval(key) - Gitlab::Redis::Cache.with do |redis| + ::Gitlab::Redis::RateLimiting.with do |redis| cache_key = action_key(key, scope) value = redis.incr(cache_key) redis.expire(cache_key, interval_value) if value == 1 @@ -154,5 +171,9 @@ module Gitlab scoped_user.username.downcase.in?(options[:users_allowlist]) end end + + private + + attr_reader :key, :options end end diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index 08214bbd449..1a9259a4f0e 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -30,7 +30,8 @@ module Gitlab end def find_sessionless_user(request_format) - find_user_from_web_access_token(request_format, scopes: [:api, :read_api]) || + find_user_from_dependency_proxy_token || + find_user_from_web_access_token(request_format, scopes: [:api, :read_api]) || find_user_from_feed_token(request_format) || find_user_from_static_object_token(request_format) || find_user_from_basic_auth_job || @@ -82,6 +83,28 @@ module Gitlab basic_auth_personal_access_token: api_request? || git_request? } end + + def find_user_from_dependency_proxy_token + return unless dependency_proxy_request? + + token, _ = ActionController::HttpAuthentication::Token.token_and_options(current_request) + + return unless token + + user_or_deploy_token = ::DependencyProxy::AuthTokenService.user_or_deploy_token_from_jwt(token) + + # Do not return deploy tokens + # See https://gitlab.com/gitlab-org/gitlab/-/issues/342481 + return unless user_or_deploy_token.is_a?(::User) + + user_or_deploy_token + rescue ActiveRecord::RecordNotFound + nil # invalid id used return no user + end + + def dependency_proxy_request? + Gitlab::PathRegex.dependency_proxy_route_regex.match?(current_request.path) + end end end end diff --git a/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb b/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb new file mode 100644 index 00000000000..9b278efaedd --- /dev/null +++ b/lib/gitlab/background_migration/fix_first_mentioned_in_commit_at.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that fixes the incorrectly set authored_date within + # issue_metrics table + class FixFirstMentionedInCommitAt + SUB_BATCH_SIZE = 500 + + # rubocop: disable Style/Documentation + class TmpIssueMetrics < ActiveRecord::Base + include EachBatch + + self.table_name = 'issue_metrics' + + def self.from_2020 + where('EXTRACT(YEAR FROM first_mentioned_in_commit_at) > 2019') + end + end + # rubocop: enable Style/Documentation + + def perform(start_id, end_id) + scope(start_id, end_id).each_batch(of: SUB_BATCH_SIZE, column: :issue_id) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(issue_id), max(issue_id)')).first + + # The query need to be reconstructed because .each_batch modifies the default scope + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 + inner_query = TmpIssueMetrics + .unscoped + .merge(scope(first, last)) + .from("issue_metrics, #{lateral_query}") + .select('issue_metrics.issue_id', 'first_authored_date.authored_date') + .where('issue_metrics.first_mentioned_in_commit_at > first_authored_date.authored_date') + + TmpIssueMetrics.connection.execute <<~UPDATE_METRICS + WITH cte AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( + #{inner_query.to_sql} + ) + UPDATE issue_metrics + SET + first_mentioned_in_commit_at = cte.authored_date + FROM + cte + WHERE + cte.issue_id = issue_metrics.issue_id + UPDATE_METRICS + end + + mark_job_as_succeeded(start_id, end_id) + end + + private + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'FixFirstMentionedInCommitAt', + arguments + ) + end + + def scope(start_id, end_id) + TmpIssueMetrics.from_2020.where(issue_id: start_id..end_id) + end + + def lateral_query + <<~SQL + LATERAL ( + SELECT MIN(first_authored_date.authored_date) as authored_date + FROM merge_requests_closing_issues, + LATERAL ( + SELECT id + FROM merge_request_diffs + WHERE merge_request_id = merge_requests_closing_issues.merge_request_id + ORDER BY id DESC + LIMIT 1 + ) last_diff_id, + LATERAL ( + SELECT authored_date + FROM merge_request_diff_commits + WHERE + merge_request_diff_id = last_diff_id.id + ORDER BY relative_order DESC + LIMIT 1 + ) first_authored_date + WHERE merge_requests_closing_issues.issue_id = issue_metrics.issue_id + ) first_authored_date + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb index dc31f995ae0..909bf10341a 100644 --- a/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb +++ b/lib/gitlab/background_migration/populate_finding_uuid_for_vulnerability_feedback.rb @@ -38,7 +38,7 @@ module Gitlab end def vulnerability_finding - BatchLoader.for(finding_key).batch(replace_methods: false) do |finding_keys, loader| + BatchLoader.for(finding_key).batch do |finding_keys, loader| project_ids = finding_keys.map { |key| key[:project_id] } categories = finding_keys.map { |key| key[:category] } fingerprints = finding_keys.map { |key| key[:project_fingerprint] } diff --git a/lib/gitlab/background_migration/populate_status_column_of_security_scans.rb b/lib/gitlab/background_migration/populate_status_column_of_security_scans.rb new file mode 100644 index 00000000000..9740bcaa86b --- /dev/null +++ b/lib/gitlab/background_migration/populate_status_column_of_security_scans.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class PopulateStatusColumnOfSecurityScans # rubocop:disable Style/Documentation + def perform(_start_id, _end_id) + # no-op + end + end + end +end + +Gitlab::BackgroundMigration::PopulateStatusColumnOfSecurityScans.prepend_mod diff --git a/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb b/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb new file mode 100644 index 00000000000..1d96872d445 --- /dev/null +++ b/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + SUB_BATCH_SIZE = 1_000 + + # The class to populates the total projects counter cache of topics + class PopulateTopicsTotalProjectsCountCache + # Temporary AR model for topics + class Topic < ActiveRecord::Base + include EachBatch + + self.table_name = 'topics' + end + + def perform(start_id, stop_id) + Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch| + ActiveRecord::Base.connection.execute(<<~SQL) + WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql}) + UPDATE topics + SET total_projects_count = (SELECT COUNT(*) FROM project_topics WHERE topic_id = batched_relation.id) + FROM batched_relation + WHERE topics.id = batched_relation.id + SQL + end + end + end + end +end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 137f76bc96d..99ce1119c17 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -69,7 +69,7 @@ module Gitlab self.sha = commit.sha self.status = commit.status - self.ref = project.default_branch + self.ref = project.repository.root_ref end # We only cache the status for the HEAD commit of a project @@ -79,7 +79,7 @@ module Gitlab return unless sha return unless ref - if commit.sha == sha && project.default_branch == ref + if commit.sha == sha && project.repository.root_ref == ref store_in_cache end end diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 947efee43a9..4dbce0b05e1 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -84,8 +84,10 @@ module Gitlab key = cache_key_for(raw_key) Redis::Cache.with do |redis| - redis.incr(key) + value = redis.incr(key) redis.expire(key, timeout) + + value end end diff --git a/lib/gitlab/chat/command.rb b/lib/gitlab/chat/command.rb index 0add53f8174..9370c594ce1 100644 --- a/lib/gitlab/chat/command.rb +++ b/lib/gitlab/chat/command.rb @@ -66,7 +66,8 @@ module Gitlab def build_environment_variables(pipeline) pipeline.variables.build( [{ key: 'CHAT_INPUT', value: arguments }, - { key: 'CHAT_CHANNEL', value: channel }] + { key: 'CHAT_CHANNEL', value: channel }, + { key: 'CHAT_USER_ID', value: chat_name.chat_id }] ) end diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index e37cbc0442b..e5ce862264f 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -13,23 +13,21 @@ module Gitlab end def match? - if ::Gitlab::Database::LoadBalancing.enable? - # When a user merges a merge request, the following sequence happens: - # - # 1. Sidekiq: MergeService runs and updates the merge request in a locked state. - # 2. Gitaly: The UserMergeBranch RPC runs. - # 3. Gitaly (gitaly-ruby): This RPC calls the pre-receive hook. - # 4. Rails: This hook makes an API request to /api/v4/internal/allowed. - # 5. Rails: This API check does a SQL query for locked merge - # requests with a matching SHA. - # - # Since steps 1 and 5 will happen on different database - # sessions, replication lag could erroneously cause step 5 to - # report no matching merge requests. To avoid this, we check - # the write location to ensure the replica can make this query. - track_session_metrics do - ::Gitlab::Database::LoadBalancing::Sticking.select_valid_host(:project, @project.id) - end + # When a user merges a merge request, the following sequence happens: + # + # 1. Sidekiq: MergeService runs and updates the merge request in a locked state. + # 2. Gitaly: The UserMergeBranch RPC runs. + # 3. Gitaly (gitaly-ruby): This RPC calls the pre-receive hook. + # 4. Rails: This hook makes an API request to /api/v4/internal/allowed. + # 5. Rails: This API check does a SQL query for locked merge + # requests with a matching SHA. + # + # Since steps 1 and 5 will happen on different database + # sessions, replication lag could erroneously cause step 5 to + # report no matching merge requests. To avoid this, we check + # the write location to ensure the replica can make this query. + track_session_metrics do + ::ApplicationRecord.sticking.select_valid_host(:project, @project.id) end # rubocop: disable CodeReuse/ActiveRecord diff --git a/lib/gitlab/ci/badge/coverage/report.rb b/lib/gitlab/ci/badge/coverage/report.rb index 28863a0703b..78b51dbdaf0 100644 --- a/lib/gitlab/ci/badge/coverage/report.rb +++ b/lib/gitlab/ci/badge/coverage/report.rb @@ -15,7 +15,10 @@ module Gitlab::Ci @job = opts[:job] @customization = { key_width: opts[:key_width].to_i, - key_text: opts[:key_text] + key_text: opts[:key_text], + min_good: opts[:min_good].to_i, + min_acceptable: opts[:min_acceptable].to_i, + min_medium: opts[:min_medium].to_i } end diff --git a/lib/gitlab/ci/badge/coverage/template.rb b/lib/gitlab/ci/badge/coverage/template.rb index 96702420e9d..f12b4f2dbfb 100644 --- a/lib/gitlab/ci/badge/coverage/template.rb +++ b/lib/gitlab/ci/badge/coverage/template.rb @@ -16,12 +16,20 @@ module Gitlab::Ci low: '#e05d44', unknown: '#9f9f9f' }.freeze + COVERAGE_MAX = 100 + COVERAGE_MIN = 0 + MIN_GOOD_DEFAULT = 95 + MIN_ACCEPTABLE_DEFAULT = 90 + MIN_MEDIUM_DEFAULT = 75 def initialize(badge) @entity = badge.entity @status = badge.status @key_text = badge.customization.dig(:key_text) @key_width = badge.customization.dig(:key_width) + @min_good = badge.customization.dig(:min_good) + @min_acceptable = badge.customization.dig(:min_acceptable) + @min_medium = badge.customization.dig(:min_medium) end def value_text @@ -32,12 +40,36 @@ module Gitlab::Ci @status ? 54 : 58 end + def min_good_value + if @min_good && @min_good.between?(3, COVERAGE_MAX) + @min_good + else + MIN_GOOD_DEFAULT + end + end + + def min_acceptable_value + if @min_acceptable && @min_acceptable.between?(2, min_good_value - 1) + @min_acceptable + else + [MIN_ACCEPTABLE_DEFAULT, (min_good_value - 1)].min + end + end + + def min_medium_value + if @min_medium && @min_medium.between?(1, min_acceptable_value - 1) + @min_medium + else + [MIN_MEDIUM_DEFAULT, (min_acceptable_value - 1)].min + end + end + def value_color case @status - when 95..100 then STATUS_COLOR[:good] - when 90..95 then STATUS_COLOR[:acceptable] - when 75..90 then STATUS_COLOR[:medium] - when 0..75 then STATUS_COLOR[:low] + when min_good_value..COVERAGE_MAX then STATUS_COLOR[:good] + when min_acceptable_value..min_good_value then STATUS_COLOR[:acceptable] + when min_medium_value..min_acceptable_value then STATUS_COLOR[:medium] + when COVERAGE_MIN..min_medium_value then STATUS_COLOR[:low] else STATUS_COLOR[:unknown] end diff --git a/lib/gitlab/ci/build/auto_retry.rb b/lib/gitlab/ci/build/auto_retry.rb index b98d1d7b330..6ab567dff7c 100644 --- a/lib/gitlab/ci/build/auto_retry.rb +++ b/lib/gitlab/ci/build/auto_retry.rb @@ -9,7 +9,8 @@ class Gitlab::Ci::Build::AutoRetry RETRY_OVERRIDES = { ci_quota_exceeded: 0, - no_matching_runner: 0 + no_matching_runner: 0, + missing_dependency_failure: 0 }.freeze def initialize(build) diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 97e4922b2a1..95f1a842c50 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -58,9 +58,6 @@ module Gitlab end def verify_rules(location) - # Behaves like there is no `rules` - return location unless ::Feature.enabled?(:ci_include_rules, context.project, default_enabled: :yaml) - return unless Rules.new(location[:rules]).evaluate(context).pass? location diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb index 5a788427172..95470537de3 100644 --- a/lib/gitlab/ci/config/external/rules.rb +++ b/lib/gitlab/ci/config/external/rules.rb @@ -5,7 +5,13 @@ module Gitlab class Config module External class Rules + ALLOWED_KEYS = Entry::Include::Rules::Rule::ALLOWED_KEYS + + InvalidIncludeRulesError = Class.new(Mapper::Error) + def initialize(rule_hashes) + validate(rule_hashes) + @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes) end @@ -19,6 +25,16 @@ module Gitlab @rule_list.find { |rule| rule.matches?(nil, context) } end + def validate(rule_hashes) + return unless rule_hashes.is_a?(Array) + + rule_hashes.each do |rule_hash| + next if (rule_hash.keys - ALLOWED_KEYS).empty? + + raise InvalidIncludeRulesError, "invalid include rule: #{rule_hash}" + end + end + Result = Struct.new(:result) do def pass? !!result diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 27bb7fdc05a..28ba1cd4d47 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -91,7 +91,8 @@ module Gitlab email: current_user.email, created_at: current_user.created_at&.iso8601, current_sign_in_ip: current_user.current_sign_in_ip, - last_sign_in_ip: current_user.last_sign_in_ip + last_sign_in_ip: current_user.last_sign_in_ip, + sign_in_count: current_user.sign_in_count }, pipeline: { sha: pipeline.sha, diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 28df9f5386c..321efa7854f 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -65,13 +65,6 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end - def self.legacy_update_jobs_counter - name = :ci_legacy_update_jobs_as_retried_total - comment = 'Counter of occurrences when jobs were not being set as retried before update_retried' - - Gitlab::Metrics.counter(name, comment) - end - def self.pipeline_failure_reason_counter name = :gitlab_ci_pipeline_failure_reasons comment = 'Counter of pipeline failure reasons' @@ -92,14 +85,6 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end - - def self.gitlab_ci_difference_live_vs_actual_minutes - name = :gitlab_ci_difference_live_vs_actual_minutes - comment = 'Comparison between CI minutes consumption from live tracking vs actual consumption' - labels = {} - buckets = [-120.0, -60.0, -30.0, -10.0, -5.0, -3.0, -1.0, 0.0, 1.0, 3.0, 5.0, 10.0, 30.0, 60.0, 120.0] - ::Gitlab::Metrics.histogram(name, comment, labels, buckets) - end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 934bf22d8ad..9ad5d6538b7 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -106,10 +106,15 @@ module Gitlab environment = Seed::Environment.new(build).to_resource - # If there is a validation error on environment creation, such as - # the name contains invalid character, the build falls back to a - # non-environment job. unless environment.persisted? + if Feature.enabled?(:surface_environment_creation_failure, build.project, default_enabled: :yaml) && + Feature.disabled?(:surface_environment_creation_failure_override, build.project) + return { status: :failed, failure_reason: :environment_creation_failure } + end + + # If there is a validation error on environment creation, such as + # the name contains invalid character, the build falls back to a + # non-environment job. Gitlab::ErrorTracking.track_exception( EnvironmentCreationFailure.new, project_id: build.project_id, diff --git a/lib/gitlab/ci/reports/security/flag.rb b/lib/gitlab/ci/reports/security/flag.rb index 7e6cc758864..8370dd60418 100644 --- a/lib/gitlab/ci/reports/security/flag.rb +++ b/lib/gitlab/ci/reports/security/flag.rb @@ -20,7 +20,7 @@ module Gitlab @description = description end - def to_hash + def to_h { flag_type: flag_type, origin: origin, diff --git a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb index 6cb2e0ddb33..4be4cf62e7b 100644 --- a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb +++ b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb @@ -80,6 +80,8 @@ module Gitlab matcher = FindingMatcher.new(head_findings) base_findings.each do |base_finding| + next if base_finding.requires_manual_resolution? + matched_head_finding = matcher.find_and_remove_match!(base_finding) @fixed_findings << base_finding if matched_head_finding.nil? diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index ee210e51232..b0f12ff7517 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -33,7 +33,8 @@ module Gitlab ci_quota_exceeded: 'no more CI minutes available', no_matching_runner: 'no matching runner available', trace_size_exceeded: 'log size limit exceeded', - builds_disabled: 'project builds are disabled' + builds_disabled: 'project builds are disabled', + environment_creation_failure: 'environment creation failure' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index e0627b85aba..65a58130962 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.12.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.14.0' .dast-auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 2df985cfbb5..58f13746a1f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.12.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.14.0' .auto-deploy: image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 917a28bb1ee..37a746a223c 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -253,6 +253,7 @@ semgrep-sast: - '**/*.ts' - '**/*.tsx' - '**/*.c' + - '**/*.go' sobelow-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index bfea437b8f1..64c784f43cb 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -11,7 +11,7 @@ publish: changes: - package.json script: - # If no .npmrc if included in the repo, generate a temporary one that is configured to publish to GitLab's NPM registry + # If no .npmrc is included in the repo, generate a temporary one that is configured to publish to GitLab's NPM registry - | if [[ ! -f .npmrc ]]; then echo 'No .npmrc found! Creating one now. Please review the following link for more information: https://docs.gitlab.com/ee/user/packages/npm_registry/index.html#project-level-npm-endpoint-1' diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 72a94dcd412..25075cc8f90 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -25,7 +25,7 @@ module Gitlab delegate :old_trace, to: :job delegate :can_attempt_archival_now?, :increment_archival_attempts!, - :archival_attempts_message, to: :trace_metadata + :archival_attempts_message, :archival_attempts_available?, to: :trace_metadata def initialize(job) @job = job @@ -122,6 +122,10 @@ module Gitlab end end + def attempt_archive_cleanup! + destroy_any_orphan_trace_data! + end + def update_interval if being_watched? UPDATE_FREQUENCY_WHEN_BEING_WATCHED @@ -191,7 +195,10 @@ module Gitlab def unsafe_archive! raise ArchiveError, 'Job is not finished yet' unless job.complete? - unsafe_trace_conditionally_cleanup_before_retry! + already_archived?.tap do |archived| + destroy_any_orphan_trace_data! + raise AlreadyArchivedError, 'Could not archive again' if archived + end if job.trace_chunks.any? Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream| @@ -214,16 +221,15 @@ module Gitlab def already_archived? # TODO check checksum to ensure archive completed successfully # See https://gitlab.com/gitlab-org/gitlab/-/issues/259619 - trace_artifact.archived_trace_exists? + trace_artifact&.archived_trace_exists? end - def unsafe_trace_conditionally_cleanup_before_retry! + def destroy_any_orphan_trace_data! return unless trace_artifact if already_archived? # An archive already exists, so make sure to remove the trace chunks erase_trace_chunks! - raise AlreadyArchivedError, 'Could not archive again' else # An archive already exists, but its associated file does not, so remove it trace_artifact.destroy! @@ -236,35 +242,7 @@ module Gitlab end def archive_stream!(stream) - clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| - create_build_trace!(job, clone_path) - end - end - - def clone_file!(src_stream, temp_dir) - FileUtils.mkdir_p(temp_dir) - Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path| - temp_path = File.join(dir_path, "job.log") - FileUtils.touch(temp_path) - size = IO.copy_stream(src_stream, temp_path) - raise ArchiveError, 'Failed to copy stream' unless size == src_stream.size - - yield(temp_path) - end - end - - def create_build_trace!(job, path) - File.open(path) do |stream| - # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration - # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20307 - trace_artifact = job.create_job_artifacts_trace!( - project: job.project, - file_type: :trace, - file: stream, - file_sha256: self.class.hexdigest(path)) - - trace_metadata.track_archival!(trace_artifact.id) - end + ::Gitlab::Ci::Trace::Archive.new(job, trace_metadata).execute!(stream) end def trace_metadata @@ -314,7 +292,8 @@ module Gitlab def destroy_stream(build) if consistent_archived_trace?(build) - ::Gitlab::Database::LoadBalancing::Sticking + ::Ci::Build + .sticking .stick(LOAD_BALANCING_STICKING_NAMESPACE, build.id) end @@ -323,7 +302,8 @@ module Gitlab def read_trace_artifact(build) if consistent_archived_trace?(build) - ::Gitlab::Database::LoadBalancing::Sticking + ::Ci::Build + .sticking .unstick_or_continue_sticking(LOAD_BALANCING_STICKING_NAMESPACE, build.id) end diff --git a/lib/gitlab/ci/trace/archive.rb b/lib/gitlab/ci/trace/archive.rb new file mode 100644 index 00000000000..5047cf04562 --- /dev/null +++ b/lib/gitlab/ci/trace/archive.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + class Archive + include ::Gitlab::Utils::StrongMemoize + include Checksummable + + def initialize(job, trace_metadata, metrics = ::Gitlab::Ci::Trace::Metrics.new) + @job = job + @trace_metadata = trace_metadata + @metrics = metrics + end + + def execute!(stream) + clone_file!(stream, JobArtifactUploader.workhorse_upload_path) do |clone_path| + md5_checksum = self.class.md5_hexdigest(clone_path) + sha256_checksum = self.class.sha256_hexdigest(clone_path) + + job.transaction do + self.trace_artifact = create_build_trace!(clone_path, sha256_checksum) + trace_metadata.track_archival!(trace_artifact.id, md5_checksum) + end + end + + validate_archived_trace + end + + private + + attr_reader :job, :trace_metadata, :metrics + attr_accessor :trace_artifact + + def clone_file!(src_stream, temp_dir) + FileUtils.mkdir_p(temp_dir) + Dir.mktmpdir("tmp-trace-#{job.id}", temp_dir) do |dir_path| + temp_path = File.join(dir_path, "job.log") + FileUtils.touch(temp_path) + size = IO.copy_stream(src_stream, temp_path) + raise ::Gitlab::Ci::Trace::ArchiveError, 'Failed to copy stream' unless size == src_stream.size + + yield(temp_path) + end + end + + def create_build_trace!(path, file_sha256) + File.open(path) do |stream| + # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration + # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20307 + job.create_job_artifacts_trace!( + project: job.project, + file_type: :trace, + file: stream, + file_sha256: file_sha256) + end + end + + def validate_archived_trace + return unless remote_checksum + + trace_metadata.update!(remote_checksum: remote_checksum) + + unless trace_metadata.remote_checksum_valid? + metrics.increment_error_counter(type: :archive_invalid_checksum) + end + end + + def remote_checksum + strong_memoize(:remote_checksum) do + ::Gitlab::Ci::Trace::RemoteChecksum.new(trace_artifact).md5_checksum + end + end + end + end + end +end diff --git a/lib/gitlab/ci/trace/metrics.rb b/lib/gitlab/ci/trace/metrics.rb index fcd70634630..174a5f184ff 100644 --- a/lib/gitlab/ci/trace/metrics.rb +++ b/lib/gitlab/ci/trace/metrics.rb @@ -21,6 +21,12 @@ module Gitlab :corrupted # malformed trace found after comparing CRC32 and size ].freeze + TRACE_ERROR_TYPES = [ + :chunks_invalid_size, # used to be :corrupted + :chunks_invalid_checksum, # used to be :invalid + :archive_invalid_checksum # malformed trace found into object store after comparing MD5 + ].freeze + def increment_trace_operation(operation: :unknown) unless OPERATIONS.include?(operation) raise ArgumentError, "unknown trace operation: #{operation}" @@ -33,6 +39,14 @@ module Gitlab self.class.trace_bytes.increment({}, size.to_i) end + def increment_error_counter(type: :unknown) + unless TRACE_ERROR_TYPES.include?(type) + raise ArgumentError, "unknown error type: #{type}" + end + + self.class.trace_errors_counter.increment(type: type) + end + def observe_migration_duration(seconds) self.class.finalize_histogram.observe({}, seconds.to_f) end @@ -65,6 +79,15 @@ module Gitlab ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end end + + def self.trace_errors_counter + strong_memoize(:trace_errors_counter) do + name = :gitlab_ci_build_trace_errors_total + comment = 'Total amount of different error types on a build trace' + + Gitlab::Metrics.counter(name, comment) + end + end end end end diff --git a/lib/gitlab/ci/trace/remote_checksum.rb b/lib/gitlab/ci/trace/remote_checksum.rb new file mode 100644 index 00000000000..d57f3888ec0 --- /dev/null +++ b/lib/gitlab/ci/trace/remote_checksum.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Trace + ## + # RemoteChecksum class is responsible for fetching the MD5 checksum of + # an uploaded build trace. + # + class RemoteChecksum + include Gitlab::Utils::StrongMemoize + + def initialize(trace_artifact) + @trace_artifact = trace_artifact + end + + def md5_checksum + strong_memoize(:md5_checksum) do + fetch_md5_checksum + end + end + + private + + attr_reader :trace_artifact + delegate :aws?, :google?, to: :object_store_config, prefix: :provider + + def fetch_md5_checksum + return unless Feature.enabled?(:ci_archived_build_trace_checksum, trace_artifact.project, default_enabled: :yaml) + return unless object_store_config.enabled? + return if trace_artifact.local_store? + + remote_checksum_value + end + + def remote_checksum_value + strong_memoize(:remote_checksum_value) do + if provider_google? + checksum_from_google + elsif provider_aws? + checksum_from_aws + end + end + end + + def object_store_config + strong_memoize(:object_store_config) do + trace_artifact.file.class.object_store_config + end + end + + def checksum_from_google + content_md5 = upload_attributes.fetch(:content_md5) + + Base64 + .decode64(content_md5) + .unpack1('H*') + end + + def checksum_from_aws + upload_attributes.fetch(:etag) + end + + # Carrierwave caches attributes for the local file and does not replace + # them with the ones from object store after the upload completes. + # We need to force it to fetch them directly from the object store. + def upload_attributes + strong_memoize(:upload_attributes) do + ::Ci::JobArtifact.find(trace_artifact.id).file.file.attributes + end + end + end + end + end +end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index bdcedd1896d..0e3fa8b8d87 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -35,6 +35,10 @@ module Gitlab # However Safari seems to read child-src first so we'll just keep both equal directives['child_src'] = directives['frame_src'] + # connect_src with 'self' includes https/wss variations of the origin, + # however, safari hasn't covered this yet and we need to explicitly add + # support for websocket origins until Safari catches up with the specs + allow_websocket_connections(directives) allow_webpack_dev_server(directives) if Rails.env.development? allow_cdn(directives, Settings.gitlab.cdn_host) if Settings.gitlab.cdn_host.present? allow_customersdot(directives) if Rails.env.development? && ENV['CUSTOMER_PORTAL_URL'].present? @@ -67,6 +71,22 @@ module Gitlab arguments.strip.split(' ').map(&:strip) end + def self.allow_websocket_connections(directives) + http_ports = [80, 443] + host = Gitlab.config.gitlab.host + port = Gitlab.config.gitlab.port + secure = Gitlab.config.gitlab.https + protocol = secure ? 'wss' : 'ws' + + ws_url = "#{protocol}://#{host}" + + unless http_ports.include?(port) + ws_url = "#{ws_url}:#{port}" + end + + append_to_directive(directives, 'connect_src', ws_url) + end + def self.allow_webpack_dev_server(directives) secure = Settings.webpack.dev_server['https'] host_and_port = "#{Settings.webpack.dev_server['host']}:#{Settings.webpack.dev_server['port']}" diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 385ac40cf13..b560d4cbca8 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -49,18 +49,29 @@ module Gitlab # It does not include the default public schema EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze - DATABASES = ActiveRecord::Base - .connection_handler - .connection_pools - .each_with_object({}) do |pool, hash| - hash[pool.db_config.name.to_sym] = Connection.new(pool.connection_klass) - end - .freeze - PRIMARY_DATABASE_NAME = ActiveRecord::Base.connection_db_config.name.to_sym + def self.database_base_models + @database_base_models ||= { + # Note that we use ActiveRecord::Base here and not ApplicationRecord. + # This is deliberate, as we also use these classes to apply load + # balancing to, and the load balancer must be enabled for _all_ models + # that inher from ActiveRecord::Base; not just our own models that + # inherit from ApplicationRecord. + main: ::ActiveRecord::Base, + ci: ::Ci::CiDatabaseRecord.connection_class? ? ::Ci::CiDatabaseRecord : nil + }.compact.freeze + end + + def self.databases + @databases ||= database_base_models + .transform_values { |connection_class| Connection.new(connection_class) } + .with_indifferent_access + .freeze + end + def self.main - DATABASES[PRIMARY_DATABASE_NAME] + databases[PRIMARY_DATABASE_NAME] end # We configure the database connection pool size automatically based on the @@ -99,7 +110,7 @@ module Gitlab def self.check_postgres_version_and_print_warning return if Gitlab::Runtime.rails_runner? - DATABASES.each do |name, connection| + databases.each do |name, connection| next if connection.postgresql_minimum_supported_version? Kernel.warn ERB.new(Rainbow.new.wrap(<<~EOS).red).result @@ -111,7 +122,7 @@ module Gitlab ███ ███ ██ ██ ██ ██ ██ ████ ██ ██ ████ ██████ ****************************************************************************** - You are using PostgreSQL <%= Gitlab::Database.main.version %> for the #{name} database, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %> + You are using PostgreSQL #{connection.version} for the #{name} database, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %> is required for this version of GitLab. <% if Rails.env.development? || Rails.env.test? %> If using gitlab-development-kit, please find the relevant steps here: diff --git a/lib/gitlab/database/count.rb b/lib/gitlab/database/count.rb index eac61254bdf..ce61c1ba9ad 100644 --- a/lib/gitlab/database/count.rb +++ b/lib/gitlab/database/count.rb @@ -35,7 +35,17 @@ module Gitlab # # @param [Array] # @return [Hash] of Model -> count mapping - def self.approximate_counts(models, strategies: [TablesampleCountStrategy, ReltuplesCountStrategy, ExactCountStrategy]) + def self.approximate_counts(models, strategies: []) + if strategies.empty? + # ExactCountStrategy is the only strategy working on read-only DBs, as others make + # use of tuple stats which use the primary DB to estimate tables size in a transaction. + strategies = if ::Gitlab::Database.read_write? + [TablesampleCountStrategy, ReltuplesCountStrategy, ExactCountStrategy] + else + [ExactCountStrategy] + end + end + strategies.each_with_object({}) do |strategy, counts_by_model| models_with_missing_counts = models - counts_by_model.keys diff --git a/lib/gitlab/database/load_balancing.rb b/lib/gitlab/database/load_balancing.rb index bbfbf83222f..3e322e752b7 100644 --- a/lib/gitlab/database/load_balancing.rb +++ b/lib/gitlab/database/load_balancing.rb @@ -4,72 +4,34 @@ module Gitlab module Database module LoadBalancing # The exceptions raised for connection errors. - CONNECTION_ERRORS = if defined?(PG) - [ - PG::ConnectionBad, - PG::ConnectionDoesNotExist, - PG::ConnectionException, - PG::ConnectionFailure, - PG::UnableToSend, - # During a failover this error may be raised when - # writing to a primary. - PG::ReadOnlySqlTransaction - ].freeze - else - [].freeze - end - - ProxyNotConfiguredError = Class.new(StandardError) - - # The connection proxy to use for load balancing (if enabled). - def self.proxy - unless load_balancing_proxy = ActiveRecord::Base.load_balancing_proxy - Gitlab::ErrorTracking.track_exception( - ProxyNotConfiguredError.new( - "Attempting to access the database load balancing proxy, but it wasn't configured.\n" \ - "Did you forget to call '#{self.name}.configure_proxy'?" - )) - end - - load_balancing_proxy - end - - # Returns a Hash containing the load balancing configuration. - def self.configuration - @configuration ||= Configuration.for_model(ActiveRecord::Base) - end - - # Returns true if load balancing is to be enabled. - def self.enable? - return false if Gitlab::Runtime.rake? - - configured? - end + CONNECTION_ERRORS = [ + PG::ConnectionBad, + PG::ConnectionDoesNotExist, + PG::ConnectionException, + PG::ConnectionFailure, + PG::UnableToSend, + # During a failover this error may be raised when + # writing to a primary. + PG::ReadOnlySqlTransaction, + # This error is raised when we can't connect to the database in the + # first place (e.g. it's offline or the hostname is incorrect). + ActiveRecord::ConnectionNotEstablished + ].freeze - def self.configured? - configuration.load_balancing_enabled? || - configuration.service_discovery_enabled? + def self.base_models + @base_models ||= ::Gitlab::Database.database_base_models.values.freeze end - def self.start_service_discovery - return unless configuration.service_discovery_enabled? + def self.each_load_balancer + return to_enum(__method__) unless block_given? - ServiceDiscovery - .new(proxy.load_balancer, **configuration.service_discovery) - .start + base_models.each do |model| + yield model.connection.load_balancer + end end - # Configures proxying of requests. - def self.configure_proxy - lb = LoadBalancer.new(configuration, primary_only: !enable?) - ActiveRecord::Base.load_balancing_proxy = ConnectionProxy.new(lb) - - # Populate service discovery immediately if it is configured - if configuration.service_discovery_enabled? - ServiceDiscovery - .new(lb, **configuration.service_discovery) - .perform_service_discovery - end + def self.release_hosts + each_load_balancer(&:release_host) end DB_ROLES = [ diff --git a/lib/gitlab/database/load_balancing/action_cable_callbacks.rb b/lib/gitlab/database/load_balancing/action_cable_callbacks.rb index 4feba989a0a..7164976ff73 100644 --- a/lib/gitlab/database/load_balancing/action_cable_callbacks.rb +++ b/lib/gitlab/database/load_balancing/action_cable_callbacks.rb @@ -16,7 +16,7 @@ module Gitlab inner.call ensure - ::Gitlab::Database::LoadBalancing.proxy.load_balancer.release_host + ::Gitlab::Database::LoadBalancing.release_hosts ::Gitlab::Database::LoadBalancing::Session.clear_session end end diff --git a/lib/gitlab/database/load_balancing/active_record_proxy.rb b/lib/gitlab/database/load_balancing/active_record_proxy.rb deleted file mode 100644 index deaea62d774..00000000000 --- a/lib/gitlab/database/load_balancing/active_record_proxy.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module LoadBalancing - # Module injected into ActiveRecord::Base to allow hijacking of the - # "connection" method. - module ActiveRecordProxy - def connection - ::Gitlab::Database::LoadBalancing.proxy - end - end - end - end -end diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index 238f55fd98e..6156515bd73 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -72,7 +72,14 @@ module Gitlab Database.default_pool_size end + # Returns `true` if the use of load balancing replicas should be + # enabled. + # + # This is disabled for Rake tasks to ensure e.g. database migrations + # always produce consistent results. def load_balancing_enabled? + return false if Gitlab::Runtime.rake? + hosts.any? || service_discovery_enabled? end diff --git a/lib/gitlab/database/load_balancing/host.rb b/lib/gitlab/database/load_balancing/host.rb index acd7df0a263..bdbb80d6f31 100644 --- a/lib/gitlab/database/load_balancing/host.rb +++ b/lib/gitlab/database/load_balancing/host.rb @@ -9,19 +9,12 @@ module Gitlab delegate :connection, :release_connection, :enable_query_cache!, :disable_query_cache!, :query_cache_enabled, to: :pool - CONNECTION_ERRORS = - if defined?(PG) - [ - ActionView::Template::Error, - ActiveRecord::StatementInvalid, - PG::Error - ].freeze - else - [ - ActionView::Template::Error, - ActiveRecord::StatementInvalid - ].freeze - end + CONNECTION_ERRORS = [ + ActionView::Template::Error, + ActiveRecord::StatementInvalid, + ActiveRecord::ConnectionNotEstablished, + PG::Error + ].freeze # host - The address of the database. # load_balancer - The LoadBalancer that manages this Host. diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 9b00b323301..cc9ca325337 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -12,22 +12,26 @@ module Gitlab REPLICA_SUFFIX = '_replica' - attr_reader :host_list, :configuration + attr_reader :name, :host_list, :configuration # configuration - An instance of `LoadBalancing::Configuration` that # contains the configuration details (such as the hosts) # for this load balancer. - # primary_only - If set, the replicas are ignored and the primary is - # always used. - def initialize(configuration, primary_only: false) + def initialize(configuration) @configuration = configuration - @primary_only = primary_only + @primary_only = !configuration.load_balancing_enabled? @host_list = - if primary_only + if @primary_only HostList.new([PrimaryHost.new(self)]) else HostList.new(configuration.hosts.map { |addr| Host.new(addr, self) }) end + + @name = @configuration.model.connection_db_config.name.to_sym + end + + def primary_only? + @primary_only end def disconnect!(timeout: 120) @@ -151,6 +155,17 @@ module Gitlab # Yields a block, retrying it upon error using an exponential backoff. def retry_with_backoff(retries = 3, time = 2) + # In CI we only use the primary, but databases may not always be + # available (or take a few seconds to become available). Retrying in + # this case can slow down CI jobs. In addition, retrying with _only_ + # a primary being present isn't all that helpful. + # + # To prevent this from happening, we don't make any attempt at + # retrying unless one or more replicas are used. This matches the + # behaviour from before we enabled load balancing code even if no + # replicas were configured. + return yield if primary_only? + retried = 0 last_error = nil @@ -176,6 +191,11 @@ module Gitlab def connection_error?(error) case error + when ActiveRecord::NoDatabaseError + # Retrying this error isn't going to magically make the database + # appear. It also slows down CI jobs that are meant to create the + # database in the first place. + false when ActiveRecord::StatementInvalid, ActionView::Template::Error # After connecting to the DB Rails will wrap query errors using this # class. @@ -235,7 +255,7 @@ module Gitlab @configuration.model.connection_specification_name, role: ActiveRecord::Base.writing_role, shard: ActiveRecord::Base.default_shard - ) + ) || raise(::ActiveRecord::ConnectionNotEstablished) end private diff --git a/lib/gitlab/database/load_balancing/primary_host.rb b/lib/gitlab/database/load_balancing/primary_host.rb index e379652c260..7070cc54d4b 100644 --- a/lib/gitlab/database/load_balancing/primary_host.rb +++ b/lib/gitlab/database/load_balancing/primary_host.rb @@ -11,6 +11,12 @@ module Gitlab # balancing is enabled, but no replicas have been configured (= the # default case). class PrimaryHost + WAL_ERROR_MESSAGE = <<~MSG.strip + Obtaining WAL information when not using any replicas results in + redundant queries, and may break installations that don't support + streaming replication (e.g. AWS' Aurora database). + MSG + def initialize(load_balancer) @load_balancer = load_balancer end @@ -51,30 +57,16 @@ module Gitlab end def primary_write_location - @load_balancer.primary_write_location + raise NotImplementedError, WAL_ERROR_MESSAGE end def database_replica_location - row = query_and_release(<<-SQL.squish) - SELECT pg_last_wal_replay_lsn()::text AS location - SQL - - row['location'] if row.any? - rescue *Host::CONNECTION_ERRORS - nil + raise NotImplementedError, WAL_ERROR_MESSAGE end def caught_up?(_location) true end - - def query_and_release(sql) - connection.select_all(sql).first || {} - rescue StandardError - {} - ensure - release_connection - end end end end diff --git a/lib/gitlab/database/load_balancing/rack_middleware.rb b/lib/gitlab/database/load_balancing/rack_middleware.rb index f8a31622b7d..7ce7649cc22 100644 --- a/lib/gitlab/database/load_balancing/rack_middleware.rb +++ b/lib/gitlab/database/load_balancing/rack_middleware.rb @@ -9,23 +9,6 @@ module Gitlab class RackMiddleware STICK_OBJECT = 'load_balancing.stick_object' - # Unsticks or continues sticking the current request. - # - # This method also updates the Rack environment so #call can later - # determine if we still need to stick or not. - # - # env - The Rack environment. - # namespace - The namespace to use for sticking. - # id - The identifier to use for sticking. - def self.stick_or_unstick(env, namespace, id) - return unless ::Gitlab::Database::LoadBalancing.enable? - - ::Gitlab::Database::LoadBalancing::Sticking.unstick_or_continue_sticking(namespace, id) - - env[STICK_OBJECT] ||= Set.new - env[STICK_OBJECT] << [namespace, id] - end - def initialize(app) @app = app end @@ -53,41 +36,46 @@ module Gitlab # Typically this code will only be reachable for Rails requests as # Grape data is not yet available at this point. def unstick_or_continue_sticking(env) - namespaces_and_ids = sticking_namespaces_and_ids(env) + namespaces_and_ids = sticking_namespaces(env) - namespaces_and_ids.each do |namespace, id| - ::Gitlab::Database::LoadBalancing::Sticking.unstick_or_continue_sticking(namespace, id) + namespaces_and_ids.each do |(model, namespace, id)| + model.sticking.unstick_or_continue_sticking(namespace, id) end end # Determine if we need to stick after handling a request. def stick_if_necessary(env) - namespaces_and_ids = sticking_namespaces_and_ids(env) + namespaces_and_ids = sticking_namespaces(env) - namespaces_and_ids.each do |namespace, id| - ::Gitlab::Database::LoadBalancing::Sticking.stick_if_necessary(namespace, id) + namespaces_and_ids.each do |model, namespace, id| + model.sticking.stick_if_necessary(namespace, id) end end def clear - load_balancer.release_host + ::Gitlab::Database::LoadBalancing.release_hosts ::Gitlab::Database::LoadBalancing::Session.clear_session end - def load_balancer - ::Gitlab::Database::LoadBalancing.proxy.load_balancer - end - # Determines the sticking namespace and identifier based on the Rack # environment. # # For Rails requests this uses warden, but Grape and others have to # manually set the right environment variable. - def sticking_namespaces_and_ids(env) + def sticking_namespaces(env) warden = env['warden'] if warden && warden.user - [[:user, warden.user.id]] + # When sticking per user, _only_ sticking the main connection could + # result in the application trying to read data from a different + # connection, while that data isn't available yet. + # + # To prevent this from happening, we scope sticking to all the + # models that support load balancing. In the future (if we + # determined this to be OK) we may be able to relax this. + ::Gitlab::Database::LoadBalancing.base_models.map do |model| + [model, :user, warden.user.id] + end elsif env[STICK_OBJECT].present? env[STICK_OBJECT].to_a else diff --git a/lib/gitlab/database/load_balancing/setup.rb b/lib/gitlab/database/load_balancing/setup.rb new file mode 100644 index 00000000000..3cce839a960 --- /dev/null +++ b/lib/gitlab/database/load_balancing/setup.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + # Class for setting up load balancing of a specific model. + class Setup + attr_reader :configuration + + def initialize(model, start_service_discovery: false) + @model = model + @configuration = Configuration.for_model(model) + @start_service_discovery = start_service_discovery + end + + def setup + disable_prepared_statements + setup_load_balancer + setup_service_discovery + end + + def disable_prepared_statements + db_config_object = @model.connection_db_config + config = + db_config_object.configuration_hash.merge(prepared_statements: false) + + hash_config = ActiveRecord::DatabaseConfigurations::HashConfig.new( + db_config_object.env_name, + db_config_object.name, + config + ) + + @model.establish_connection(hash_config) + end + + def setup_load_balancer + lb = LoadBalancer.new(configuration) + + # We just use a simple `class_attribute` here so we don't need to + # inject any modules and/or expose unnecessary methods. + @model.class_attribute(:connection) + @model.class_attribute(:sticking) + + @model.connection = ConnectionProxy.new(lb) + @model.sticking = Sticking.new(lb) + end + + def setup_service_discovery + return unless configuration.service_discovery_enabled? + + lb = @model.connection.load_balancer + sv = ServiceDiscovery.new(lb, **configuration.service_discovery) + + sv.perform_service_discovery + + sv.start if @start_service_discovery + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb index 518a812b406..62dfe75a851 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb @@ -30,26 +30,26 @@ module Gitlab end def set_data_consistency_locations!(job) - # Once we add support for multiple databases to our load balancer, we would use something like this: - # job['wal_locations'] = Gitlab::Database::DATABASES.transform_values do |connection| - # connection.load_balancer.primary_write_location - # end - # - job['wal_locations'] = { Gitlab::Database::MAIN_DATABASE_NAME.to_sym => wal_location } if wal_location - end + locations = {} - def wal_location - strong_memoize(:wal_location) do - if Session.current.use_primary? - load_balancer.primary_write_location - else - load_balancer.host.database_replica_location + ::Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + if (location = wal_location_for(lb)) + locations[lb.name] = location end end + + job['wal_locations'] = locations end - def load_balancer - LoadBalancing.proxy.load_balancer + def wal_location_for(load_balancer) + # When only using the primary there's no need for any WAL queries. + return if load_balancer.primary_only? + + if ::Gitlab::Database::LoadBalancing::Session.current.use_primary? + load_balancer.primary_write_location + else + load_balancer.host.database_replica_location + end end end end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 15f8f0fb240..f0c7016032b 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -29,7 +29,7 @@ module Gitlab private def clear - release_hosts + LoadBalancing.release_hosts Session.clear_session end @@ -44,7 +44,7 @@ module Gitlab return :primary_no_wal unless wal_locations - if all_databases_has_replica_caught_up?(wal_locations) + if databases_in_sync?(wal_locations) # Happy case: we can read from a replica. retried_before?(worker_class, job) ? :replica_retried : :replica elsif can_retry?(worker_class, job) @@ -89,27 +89,18 @@ module Gitlab job['retry_count'].nil? end - def all_databases_has_replica_caught_up?(wal_locations) - wal_locations.all? do |_config_name, location| - # Once we add support for multiple databases to our load balancer, we would use something like this: - # Gitlab::Database::DATABASES[config_name].load_balancer.select_up_to_date_host(location) - load_balancer.select_up_to_date_host(location) + def databases_in_sync?(wal_locations) + LoadBalancing.each_load_balancer.all? do |lb| + if (location = wal_locations[lb.name]) + lb.select_up_to_date_host(location) + else + # If there's no entry for a load balancer it means the Sidekiq + # job doesn't care for it. In this case we'll treat the load + # balancer as being in sync. + true + end end end - - def release_hosts - # Once we add support for multiple databases to our load balancer, we would use something like this: - # connection.load_balancer.primary_write_location - # - # Gitlab::Database::DATABASES.values.each do |connection| - # connection.load_balancer.release_host - # end - load_balancer.release_host - end - - def load_balancer - LoadBalancing.proxy.load_balancer - end end end end diff --git a/lib/gitlab/database/load_balancing/sticking.rb b/lib/gitlab/database/load_balancing/sticking.rb index 20d42b9a694..df4ad18581f 100644 --- a/lib/gitlab/database/load_balancing/sticking.rb +++ b/lib/gitlab/database/load_balancing/sticking.rb @@ -5,36 +5,47 @@ module Gitlab module LoadBalancing # Module used for handling sticking connections to a primary, if # necessary. - # - # ## Examples - # - # Sticking a user to the primary: - # - # Sticking.stick_if_necessary(:user, current_user.id) - # - # To unstick if possible, or continue using the primary otherwise: - # - # Sticking.unstick_or_continue_sticking(:user, current_user.id) - module Sticking + class Sticking # The number of seconds after which a session should stop reading from # the primary. EXPIRATION = 30 - # Sticks to the primary if a write was performed. - def self.stick_if_necessary(namespace, id) - return unless LoadBalancing.enable? + def initialize(load_balancer) + @load_balancer = load_balancer + @model = load_balancer.configuration.model + end - stick(namespace, id) if Session.current.performed_write? + # Unsticks or continues sticking the current request. + # + # This method also updates the Rack environment so #call can later + # determine if we still need to stick or not. + # + # env - The Rack environment. + # namespace - The namespace to use for sticking. + # id - The identifier to use for sticking. + # model - The ActiveRecord model to scope sticking to. + def stick_or_unstick_request(env, namespace, id) + unstick_or_continue_sticking(namespace, id) + + env[RackMiddleware::STICK_OBJECT] ||= Set.new + env[RackMiddleware::STICK_OBJECT] << [@model, namespace, id] + end + + # Sticks to the primary if a write was performed. + def stick_if_necessary(namespace, id) + stick(namespace, id) if ::Gitlab::Database::LoadBalancing::Session.current.performed_write? end - # Checks if we are caught-up with all the work - def self.all_caught_up?(namespace, id) + def all_caught_up?(namespace, id) location = last_write_location_for(namespace, id) return true unless location - load_balancer.select_up_to_date_host(location).tap do |found| - ActiveSupport::Notifications.instrument('caught_up_replica_pick.load_balancing', { result: found } ) + @load_balancer.select_up_to_date_host(location).tap do |found| + ActiveSupport::Notifications.instrument( + 'caught_up_replica_pick.load_balancing', + { result: found } + ) unstick(namespace, id) if found end @@ -45,7 +56,7 @@ module Gitlab # in another thread. # # Returns true if one host was selected. - def self.select_caught_up_replicas(namespace, id) + def select_caught_up_replicas(namespace, id) location = last_write_location_for(namespace, id) # Unlike all_caught_up?, we return false if no write location exists. @@ -53,95 +64,92 @@ module Gitlab # write location. If no such location exists, err on the side of caution. return false unless location - load_balancer.select_up_to_date_host(location).tap do |selected| + @load_balancer.select_up_to_date_host(location).tap do |selected| unstick(namespace, id) if selected end end # Sticks to the primary if necessary, otherwise unsticks an object (if # it was previously stuck to the primary). - def self.unstick_or_continue_sticking(namespace, id) - Session.current.use_primary! unless all_caught_up?(namespace, id) + def unstick_or_continue_sticking(namespace, id) + return if all_caught_up?(namespace, id) + + ::Gitlab::Database::LoadBalancing::Session.current.use_primary! end # Select a replica that has caught up with the primary. If one has not been # found, stick to the primary. - def self.select_valid_host(namespace, id) - replica_selected = select_caught_up_replicas(namespace, id) + def select_valid_host(namespace, id) + replica_selected = + select_caught_up_replicas(namespace, id) - Session.current.use_primary! unless replica_selected + ::Gitlab::Database::LoadBalancing::Session.current.use_primary! unless replica_selected end # Starts sticking to the primary for the given namespace and id, using # the latest WAL pointer from the primary. - def self.stick(namespace, id) - return unless LoadBalancing.enable? - + def stick(namespace, id) mark_primary_write_location(namespace, id) - Session.current.use_primary! + ::Gitlab::Database::LoadBalancing::Session.current.use_primary! end - def self.bulk_stick(namespace, ids) - return unless LoadBalancing.enable? - + def bulk_stick(namespace, ids) with_primary_write_location do |location| ids.each do |id| set_write_location_for(namespace, id, location) end end - Session.current.use_primary! + ::Gitlab::Database::LoadBalancing::Session.current.use_primary! end - def self.with_primary_write_location - return unless LoadBalancing.configured? + def with_primary_write_location + # When only using the primary, there's no point in getting write + # locations, as the primary is always in sync with itself. + return if @load_balancer.primary_only? - # Load balancing could be enabled for the Web application server, - # but it's not activated for Sidekiq. We should update Redis with - # the write location just in case load balancing is being used. - location = - if LoadBalancing.enable? - load_balancer.primary_write_location - else - Gitlab::Database.main.get_write_location(ActiveRecord::Base.connection) - end + location = @load_balancer.primary_write_location return if location.blank? yield(location) end - def self.mark_primary_write_location(namespace, id) + def mark_primary_write_location(namespace, id) with_primary_write_location do |location| set_write_location_for(namespace, id, location) end end - # Stops sticking to the primary. - def self.unstick(namespace, id) + def unstick(namespace, id) Gitlab::Redis::SharedState.with do |redis| redis.del(redis_key_for(namespace, id)) + redis.del(old_redis_key_for(namespace, id)) end end - def self.set_write_location_for(namespace, id, location) + def set_write_location_for(namespace, id, location) Gitlab::Redis::SharedState.with do |redis| redis.set(redis_key_for(namespace, id), location, ex: EXPIRATION) + redis.set(old_redis_key_for(namespace, id), location, ex: EXPIRATION) end end - def self.last_write_location_for(namespace, id) + def last_write_location_for(namespace, id) Gitlab::Redis::SharedState.with do |redis| - redis.get(redis_key_for(namespace, id)) + redis.get(redis_key_for(namespace, id)) || + redis.get(old_redis_key_for(namespace, id)) end end - def self.redis_key_for(namespace, id) - "database-load-balancing/write-location/#{namespace}/#{id}" + def redis_key_for(namespace, id) + name = @load_balancer.name + + "database-load-balancing/write-location/#{name}/#{namespace}/#{id}" end - def self.load_balancer - LoadBalancing.proxy.load_balancer + def old_redis_key_for(namespace, id) + "database-load-balancing/write-location/#{namespace}/#{id}" end end end diff --git a/lib/gitlab/database/migrations/background_migration_helpers.rb b/lib/gitlab/database/migrations/background_migration_helpers.rb index 19d80ba1d64..bdaf0d35a83 100644 --- a/lib/gitlab/database/migrations/background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/background_migration_helpers.rb @@ -106,7 +106,7 @@ module Gitlab final_delay = 0 batch_counter = 0 - model_class.each_batch(of: batch_size) do |relation, index| + model_class.each_batch(of: batch_size, column: primary_column_name) do |relation, index| max = relation.arel_table[primary_column_name].maximum min = relation.arel_table[primary_column_name].minimum diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index d1e55eb825c..6e5ffb74411 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -4,21 +4,21 @@ module Gitlab module Database module Migrations class Instrumentation - RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze STATS_FILENAME = 'migration-stats.json' attr_reader :observations - def initialize(observer_classes = ::Gitlab::Database::Migrations::Observers.all_observers) + def initialize(result_dir:, observer_classes: ::Gitlab::Database::Migrations::Observers.all_observers) @observer_classes = observer_classes @observations = [] + @result_dir = result_dir end def observe(version:, name:, &block) observation = Observation.new(version, name) observation.success = true - observers = observer_classes.map { |c| c.new(observation) } + observers = observer_classes.map { |c| c.new(observation, @result_dir) } exception = nil diff --git a/lib/gitlab/database/migrations/observers/migration_observer.rb b/lib/gitlab/database/migrations/observers/migration_observer.rb index 85d18abb9ef..106f8f1f829 100644 --- a/lib/gitlab/database/migrations/observers/migration_observer.rb +++ b/lib/gitlab/database/migrations/observers/migration_observer.rb @@ -5,11 +5,12 @@ module Gitlab module Migrations module Observers class MigrationObserver - attr_reader :connection, :observation + attr_reader :connection, :observation, :output_dir - def initialize(observation) + def initialize(observation, output_dir) @connection = ActiveRecord::Base.connection @observation = observation + @output_dir = output_dir end def before diff --git a/lib/gitlab/database/migrations/observers/query_details.rb b/lib/gitlab/database/migrations/observers/query_details.rb index dadacd2d2fc..8f4406e79a5 100644 --- a/lib/gitlab/database/migrations/observers/query_details.rb +++ b/lib/gitlab/database/migrations/observers/query_details.rb @@ -6,7 +6,7 @@ module Gitlab module Observers class QueryDetails < MigrationObserver def before - file_path = File.join(Instrumentation::RESULT_DIR, "#{observation.version}_#{observation.name}-query-details.json") + file_path = File.join(output_dir, "#{observation.version}_#{observation.name}-query-details.json") @file = File.open(file_path, 'wb') @writer = Oj::StreamWriter.new(@file, {}) @writer.push_array diff --git a/lib/gitlab/database/migrations/observers/query_log.rb b/lib/gitlab/database/migrations/observers/query_log.rb index e15d733d2a2..c42fd8bd23d 100644 --- a/lib/gitlab/database/migrations/observers/query_log.rb +++ b/lib/gitlab/database/migrations/observers/query_log.rb @@ -7,7 +7,7 @@ module Gitlab class QueryLog < MigrationObserver def before @logger_was = ActiveRecord::Base.logger - file_path = File.join(Instrumentation::RESULT_DIR, "#{observation.version}_#{observation.name}.log") + file_path = File.join(output_dir, "#{observation.version}_#{observation.name}.log") @logger = Logger.new(file_path) ActiveRecord::Base.logger = @logger end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb new file mode 100644 index 00000000000..b267a64256b --- /dev/null +++ b/lib/gitlab/database/migrations/runner.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + class Runner + BASE_RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze + + class << self + def up + Runner.new(direction: :up, migrations: migrations_for_up, result_dir: BASE_RESULT_DIR.join('up')) + end + + def down + Runner.new(direction: :down, migrations: migrations_for_down, result_dir: BASE_RESULT_DIR.join('down')) + end + + def migration_context + @migration_context ||= ApplicationRecord.connection.migration_context + end + + private + + def migrations_for_up + existing_versions = migration_context.get_all_versions.to_set + + migration_context.migrations.reject do |migration| + existing_versions.include?(migration.version) + end + end + + def migration_file_names_this_branch + `git diff --name-only origin/HEAD...HEAD db/post_migrate db/migrate`.split("\n") + end + + def migrations_for_down + versions_this_branch = migration_file_names_this_branch.map do |m_name| + m_name.match(%r{^db/(post_)?migrate/(\d+)}) { |m| m.captures[1]&.to_i } + end.to_set + + existing_versions = migration_context.get_all_versions.to_set + migration_context.migrations.select do |migration| + existing_versions.include?(migration.version) && versions_this_branch.include?(migration.version) + end + end + end + + attr_reader :direction, :result_dir, :migrations + + delegate :migration_context, to: :class + + def initialize(direction:, migrations:, result_dir:) + raise "Direction must be up or down" unless %i[up down].include?(direction) + + @direction = direction + @migrations = migrations + @result_dir = result_dir + end + + def run + FileUtils.mkdir_p(result_dir) + + verbose_was = ActiveRecord::Migration.verbose + ActiveRecord::Migration.verbose = true + + sorted_migrations = migrations.sort_by(&:version) + sorted_migrations.reverse! if direction == :down + + instrumentation = Instrumentation.new(result_dir: result_dir) + + sorted_migrations.each do |migration| + instrumentation.observe(version: migration.version, name: migration.name) do + ActiveRecord::Migrator.new(direction, migration_context.migrations, migration_context.schema_migration, migration.version).run + end + end + ensure + if instrumentation + File.open(File.join(result_dir, Gitlab::Database::Migrations::Instrumentation::STATS_FILENAME), 'wb+') do |io| + io << instrumentation.observations.to_json + end + end + + # We clear the cache here to mirror the cache clearing that happens at the end of `db:migrate` tasks + # This clearing makes subsequent rake tasks in the same execution pick up database schema changes caused by + # the migrations that were just executed + ApplicationRecord.clear_cache! + ActiveRecord::Migration.verbose = verbose_was + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index bbde2063c41..71fb995577a 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -14,6 +14,10 @@ module Gitlab def self.sync_partitions(models_to_sync = registered_models) MultiDatabasePartitionManager.new(models_to_sync).sync_partitions end + + def self.drop_detached_partitions + MultiDatabasePartitionDropper.new.drop_detached_partitions + end end end end diff --git a/lib/gitlab/database/partitioning/detached_partition_dropper.rb b/lib/gitlab/database/partitioning/detached_partition_dropper.rb index dc63d93fd07..3e7ddece20b 100644 --- a/lib/gitlab/database/partitioning/detached_partition_dropper.rb +++ b/lib/gitlab/database/partitioning/detached_partition_dropper.rb @@ -7,18 +7,15 @@ module Gitlab return unless Feature.enabled?(:drop_detached_partitions, default_enabled: :yaml) Gitlab::AppLogger.info(message: "Checking for previously detached partitions to drop") + Postgresql::DetachedPartition.ready_to_drop.find_each do |detached_partition| - conn.transaction do + connection.transaction do # Another process may have already dropped the table and deleted this entry next unless (detached_partition = Postgresql::DetachedPartition.lock.find_by(id: detached_partition.id)) - unless check_partition_detached?(detached_partition) - Gitlab::AppLogger.error(message: "Attempt to drop attached database partition", partition_name: detached_partition.table_name) - detached_partition.destroy! - next - end + drop_detached_partition(detached_partition.table_name) - drop_one(detached_partition) + detached_partition.destroy! end rescue StandardError => e Gitlab::AppLogger.error(message: "Failed to drop previously detached partition", @@ -30,25 +27,30 @@ module Gitlab private - def drop_one(detached_partition) - conn.transaction do - conn.execute(<<~SQL) - DROP TABLE #{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{conn.quote_table_name(detached_partition.table_name)} - SQL + def drop_detached_partition(partition_name) + partition_identifier = qualify_partition_name(partition_name) + + if partition_detached?(partition_identifier) + connection.drop_table(partition_identifier, if_exists: true) - detached_partition.destroy! + Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: partition_name) + else + Gitlab::AppLogger.error(message: "Attempt to drop attached database partition", partition_name: partition_name) end - Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: detached_partition.table_name) end - def check_partition_detached?(detached_partition) + def qualify_partition_name(table_name) + "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{table_name}" + end + + def partition_detached?(partition_identifier) # PostgresPartition checks the pg_inherits view, so our partition will only show here if it's still attached # and thus should not be dropped - !PostgresPartition.for_identifier("#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{detached_partition.table_name}").exists? + !Gitlab::Database::PostgresPartition.for_identifier(partition_identifier).exists? end - def conn - @conn ||= ApplicationRecord.connection + def connection + Postgresql::DetachedPartition.connection end end end diff --git a/lib/gitlab/database/partitioning/multi_database_partition_dropper.rb b/lib/gitlab/database/partitioning/multi_database_partition_dropper.rb new file mode 100644 index 00000000000..769b658bae4 --- /dev/null +++ b/lib/gitlab/database/partitioning/multi_database_partition_dropper.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class MultiDatabasePartitionDropper + def drop_detached_partitions + Gitlab::AppLogger.info(message: "Dropping detached postgres partitions") + + each_database_connection do |name, connection| + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::AppLogger.debug(message: "Switched database connection", connection_name: name) + + DetachedPartitionDropper.new.perform + end + end + + Gitlab::AppLogger.info(message: "Finished dropping detached postgres partitions") + end + + private + + def each_database_connection + databases.each_pair do |name, connection_wrapper| + yield name, connection_wrapper.scope.connection + end + end + + def databases + Gitlab::Database.databases + end + end + end + end +end diff --git a/lib/gitlab/database/shared_model.rb b/lib/gitlab/database/shared_model.rb index 8f256758961..f304c32d731 100644 --- a/lib/gitlab/database/shared_model.rb +++ b/lib/gitlab/database/shared_model.rb @@ -2,6 +2,7 @@ module Gitlab module Database + # This abstract class is used for models which need to exist in multiple de-composed databases. class SharedModel < ActiveRecord::Base self.abstract_class = true diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 0ba23b8ffc7..1e6d80e1100 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -200,7 +200,7 @@ module Gitlab Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight end - # Array[<Hash>] with right/left keys that contains Gitlab::Diff::Line objects which text is hightlighted + # Array[<Hash>] with right/left keys that contains Gitlab::Diff::Line objects which text is highlighted def parallel_diff_lines @parallel_diff_lines ||= Gitlab::Diff::ParallelDiff.new(self).parallelize end diff --git a/lib/gitlab/doctor/secrets.rb b/lib/gitlab/doctor/secrets.rb index 1a1e9fafb1e..44f5c97c70c 100644 --- a/lib/gitlab/doctor/secrets.rb +++ b/lib/gitlab/doctor/secrets.rb @@ -72,7 +72,7 @@ module Gitlab end def valid_attribute?(data, attr) - data.public_send(attr) # rubocop:disable GitlabSecurity/PublicSend + data.send(attr) # rubocop:disable GitlabSecurity/PublicSend true rescue OpenSSL::Cipher::CipherError, TypeError diff --git a/lib/gitlab/email/handler/create_merge_request_handler.rb b/lib/gitlab/email/handler/create_merge_request_handler.rb index df12aea1988..c723c2762c7 100644 --- a/lib/gitlab/email/handler/create_merge_request_handler.rb +++ b/lib/gitlab/email/handler/create_merge_request_handler.rb @@ -61,7 +61,7 @@ module Gitlab private def build_merge_request - MergeRequests::BuildService.new(project: project, current_user: author, params: merge_request_params).execute + ::MergeRequests::BuildService.new(project: project, current_user: author, params: merge_request_params).execute end def create_merge_request @@ -78,7 +78,7 @@ module Gitlab if merge_request.errors.any? merge_request else - MergeRequests::CreateService.new(project: project, current_user: author).create(merge_request) + ::MergeRequests::CreateService.new(project: project, current_user: author).create(merge_request) end end diff --git a/lib/gitlab/email/hook/smime_signature_interceptor.rb b/lib/gitlab/email/hook/smime_signature_interceptor.rb index fe39589d019..0b092b3e41e 100644 --- a/lib/gitlab/email/hook/smime_signature_interceptor.rb +++ b/lib/gitlab/email/hook/smime_signature_interceptor.rb @@ -22,7 +22,7 @@ module Gitlab private def certificate - @certificate ||= Gitlab::Email::Smime::Certificate.from_files(key_path, cert_path, ca_certs_path) + @certificate ||= Gitlab::X509::Certificate.from_files(key_path, cert_path, ca_certs_path) end def key_path diff --git a/lib/gitlab/email/message/in_product_marketing/base.rb b/lib/gitlab/email/message/in_product_marketing/base.rb index 96551c89837..c4895d35a14 100644 --- a/lib/gitlab/email/message/in_product_marketing/base.rb +++ b/lib/gitlab/email/message/in_product_marketing/base.rb @@ -50,7 +50,7 @@ module Gitlab def cta_link case format when :html - link_to cta_text, group_email_campaigns_url(group, track: track, series: series), target: '_blank', rel: 'noopener noreferrer' + ActionController::Base.helpers.link_to cta_text, group_email_campaigns_url(group, track: track, series: series), target: '_blank', rel: 'noopener noreferrer' else [cta_text, group_email_campaigns_url(group, track: track, series: series)].join(' >> ') end @@ -89,7 +89,7 @@ module Gitlab case format when :html links.map do |text, link| - link_to(text, link) + ActionController::Base.helpers.link_to(text, link) end else '| ' + links.map do |text, link| diff --git a/lib/gitlab/email/message/in_product_marketing/helper.rb b/lib/gitlab/email/message/in_product_marketing/helper.rb index 4780e08322a..cec0aad44a6 100644 --- a/lib/gitlab/email/message/in_product_marketing/helper.rb +++ b/lib/gitlab/email/message/in_product_marketing/helper.rb @@ -7,7 +7,6 @@ module Gitlab module Helper include ActionView::Context include ActionView::Helpers::TagHelper - include ActionView::Helpers::UrlHelper private @@ -32,7 +31,7 @@ module Gitlab def link(text, link) case format when :html - link_to text, link + ActionController::Base.helpers.link_to text, link else "#{text} (#{link})" end diff --git a/lib/gitlab/email/message/in_product_marketing/trial.rb b/lib/gitlab/email/message/in_product_marketing/trial.rb index 222046a3966..11a799886ab 100644 --- a/lib/gitlab/email/message/in_product_marketing/trial.rb +++ b/lib/gitlab/email/message/in_product_marketing/trial.rb @@ -15,7 +15,7 @@ module Gitlab def tagline [ - s_('InProductMarketing|Start a free trial of GitLab Ultimate – no CC required'), + s_('InProductMarketing|Start a free trial of GitLab Ultimate – no credit card required'), s_('InProductMarketing|Improve app security with a 30-day trial'), s_('InProductMarketing|Start with a GitLab Ultimate free trial') ][series] diff --git a/lib/gitlab/email/smime/certificate.rb b/lib/gitlab/email/smime/certificate.rb deleted file mode 100644 index 3607b95b4bc..00000000000 --- a/lib/gitlab/email/smime/certificate.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Smime - class Certificate - CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze - - attr_reader :key, :cert, :ca_certs - - def key_string - key.to_s - end - - def cert_string - cert.to_pem - end - - def ca_certs_string - ca_certs.map(&:to_pem).join('\n') unless ca_certs.blank? - end - - def self.from_strings(key_string, cert_string, ca_certs_string = nil) - key = OpenSSL::PKey::RSA.new(key_string) - cert = OpenSSL::X509::Certificate.new(cert_string) - ca_certs = load_ca_certs_bundle(ca_certs_string) - - new(key, cert, ca_certs) - end - - def self.from_files(key_path, cert_path, ca_certs_path = nil) - ca_certs_string = File.read(ca_certs_path) if ca_certs_path - - from_strings(File.read(key_path), File.read(cert_path), ca_certs_string) - end - - # Returns an array of OpenSSL::X509::Certificate objects, empty array if none found - # - # Ruby OpenSSL::X509::Certificate.new will only load the first - # certificate if a bundle is presented, this allows to parse multiple certs - # in the same file - def self.load_ca_certs_bundle(ca_certs_string) - return [] unless ca_certs_string - - ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string| - OpenSSL::X509::Certificate.new(ca_cert_string) - end - end - - def initialize(key, cert, ca_certs = nil) - @key = key - @cert = cert - @ca_certs = ca_certs - end - end - end - end -end diff --git a/lib/gitlab/endpoint_attributes.rb b/lib/gitlab/endpoint_attributes.rb new file mode 100644 index 00000000000..2455e5e599f --- /dev/null +++ b/lib/gitlab/endpoint_attributes.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module EndpointAttributes + extend ActiveSupport::Concern + include Gitlab::ClassAttributes + + DEFAULT_URGENCY = Config::REQUEST_URGENCIES.fetch(:default) + + class_methods do + def feature_category(category, actions = []) + endpoint_attributes.set(actions, feature_category: category) + end + + def feature_category_for_action(action) + category = endpoint_attributes.attribute_for_action(action, :feature_category) + category || superclass_feature_category_for_action(action) + end + + def urgency(urgency_name, actions = []) + endpoint_attributes.set(actions, urgency: urgency_name) + end + + def urgency_for_action(action) + urgency = endpoint_attributes.attribute_for_action(action, :urgency) + urgency || superclass_urgency_for_action(action) || DEFAULT_URGENCY + end + + private + + def endpoint_attributes + class_attributes[:endpoint_attributes_config] ||= Config.new + end + + def superclass_feature_category_for_action(action) + return unless superclass.respond_to?(:feature_category_for_action) + + superclass.feature_category_for_action(action) + end + + def superclass_urgency_for_action(action) + return unless superclass.respond_to?(:urgency_for_action) + + superclass.urgency_for_action(action) + end + end + end +end diff --git a/lib/gitlab/endpoint_attributes/config.rb b/lib/gitlab/endpoint_attributes/config.rb new file mode 100644 index 00000000000..e31a3095736 --- /dev/null +++ b/lib/gitlab/endpoint_attributes/config.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module EndpointAttributes + class Config + RequestUrgency = Struct.new(:name, :duration) + REQUEST_URGENCIES = [ + RequestUrgency.new(:high, 0.25), + RequestUrgency.new(:medium, 0.5), + RequestUrgency.new(:default, 1), + RequestUrgency.new(:low, 5) + ].index_by(&:name).freeze + SUPPORTED_ATTRIBUTES = %i[feature_category urgency].freeze + + def initialize + @default_attributes = {} + @action_attributes = {} + end + + def defined_actions + @action_attributes.keys + end + + def set(actions, attributes) + sanitize_attributes!(attributes) + + if actions.empty? + conflicted = conflicted_attributes(attributes, @default_attributes) + raise ArgumentError, "Attributes already defined: #{conflicted.join(", ")}" if conflicted.present? + + @default_attributes.merge!(attributes) + else + set_attributes_for_actions(actions, attributes) + end + + nil + end + + def attribute_for_action(action, attribute_name) + value = @action_attributes.dig(action.to_s, attribute_name) || @default_attributes[attribute_name] + # Translate urgency to a representative struct + value = REQUEST_URGENCIES[value] if attribute_name == :urgency + value + end + + private + + def sanitize_attributes!(attributes) + unsupported_attributes = (attributes.keys - SUPPORTED_ATTRIBUTES).present? + raise ArgumentError, "Attributes not supported: #{unsupported_attributes.join(", ")}" if unsupported_attributes + + if attributes[:urgency].present? && !REQUEST_URGENCIES.key?(attributes[:urgency]) + raise ArgumentError, "Urgency not supported: #{attributes[:urgency]}" + end + end + + def set_attributes_for_actions(actions, attributes) + conflicted = conflicted_attributes(attributes, @default_attributes) + if conflicted.present? + raise ArgumentError, "#{conflicted.join(", ")} are already defined for all actions, but re-defined for #{actions.join(", ")}" + end + + actions.each do |action| + action = action.to_s + if @action_attributes[action].blank? + @action_attributes[action] = attributes.dup + else + conflicted = conflicted_attributes(attributes, @action_attributes[action]) + raise ArgumentError, "Attributes re-defined for action #{action}: #{conflicted.join(", ")}" if conflicted.present? + + @action_attributes[action].merge!(attributes) + end + end + end + + def conflicted_attributes(attributes, existing_attributes) + attributes.keys.filter { |attr| existing_attributes[attr].present? && existing_attributes[attr] != attributes[attr] } + end + end + end +end diff --git a/lib/gitlab/error_tracking/detailed_error.rb b/lib/gitlab/error_tracking/detailed_error.rb index d0b3fc176aa..d9ddb6caeec 100644 --- a/lib/gitlab/error_tracking/detailed_error.rb +++ b/lib/gitlab/error_tracking/detailed_error.rb @@ -22,6 +22,7 @@ module Gitlab :gitlab_issue, :gitlab_project, :id, + :integrated, :last_release_last_commit, :last_release_short_version, :last_release_version, diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index c74bd8e75ef..c2009628c56 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -40,10 +40,6 @@ module Gitlab }, invite_members_new_dropdown: { tracking_category: 'Growth::Expansion::Experiment::InviteMembersNewDropdown' - }, - show_trial_status_in_sidebar: { - tracking_category: 'Growth::Conversion::Experiment::ShowTrialStatusInSidebar', - rollout_strategy: :group } }.freeze diff --git a/lib/gitlab/feature_categories.rb b/lib/gitlab/feature_categories.rb new file mode 100644 index 00000000000..d06f3b14fed --- /dev/null +++ b/lib/gitlab/feature_categories.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + class FeatureCategories + FEATURE_CATEGORY_DEFAULT = 'unknown' + + attr_reader :categories + + def self.default + @default ||= self.load_from_yaml + end + + def self.load_from_yaml + categories = YAML.load_file(Rails.root.join('config', 'feature_categories.yml')) + + new(categories) + end + + def initialize(categories) + @categories = categories.to_set + end + + # If valid, returns a feature category from the given request. + def from_request(request) + category = request.headers["HTTP_X_GITLAB_FEATURE_CATEGORY"].presence + + return unless category && valid?(category) + + return unless ::Gitlab::RequestForgeryProtection.verified?(request.env) + + category + end + + def valid?(category) + categories.include?(category.to_s) + end + end +end diff --git a/lib/gitlab/form_builders/gitlab_ui_form_builder.rb b/lib/gitlab/form_builders/gitlab_ui_form_builder.rb index a5290508e42..3f9053d4e0c 100644 --- a/lib/gitlab/form_builders/gitlab_ui_form_builder.rb +++ b/lib/gitlab/form_builders/gitlab_ui_form_builder.rb @@ -22,29 +22,53 @@ module Gitlab format_options(checkbox_options, ['custom-control-input']), checked_value, unchecked_value - ) + - @template.label( - @object_name, method, format_options(label_options, ['custom-control-label']) - ) do - if help_text - @template.content_tag( - :span, - label - ) + - @template.content_tag( - :p, - help_text, - class: 'help-text' - ) - else - label - end - end + ) + generic_label(method, label, label_options, help_text: help_text) + end + end + + def gitlab_ui_radio_component( + method, + value, + label, + help_text: nil, + radio_options: {}, + label_options: {} + ) + @template.content_tag( + :div, + class: 'gl-form-radio custom-control custom-radio' + ) do + @template.radio_button( + @object_name, + method, + value, + format_options(radio_options, ['custom-control-input']) + ) + generic_label(method, label, label_options, help_text: help_text, value: value) end end private + def generic_label(method, label, label_options, help_text: nil, value: nil) + @template.label( + @object_name, method, format_options(label_options.merge({ value: value }), ['custom-control-label']) + ) do + if help_text + @template.content_tag( + :span, + label + ) + + @template.content_tag( + :p, + help_text, + class: 'help-text' + ) + else + label + end + end + end + def format_options(options, classes) classes << options[:class] diff --git a/lib/gitlab/git/keep_around.rb b/lib/gitlab/git/keep_around.rb index b6fc335c979..38f0e47c4c7 100644 --- a/lib/gitlab/git/keep_around.rb +++ b/lib/gitlab/git/keep_around.rb @@ -19,7 +19,7 @@ module Gitlab end def execute(shas) - shas.each do |sha| + shas.uniq.each do |sha| next unless sha.present? && commit_by(oid: sha) next if kept_around?(sha) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index bc15bd367d8..473bc04661c 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -870,9 +870,9 @@ module Gitlab end end - def squash(user, squash_id, start_sha:, end_sha:, author:, message:) + def squash(user, start_sha:, end_sha:, author:, message:) wrapped_gitaly_errors do - gitaly_operation_client.user_squash(user, squash_id, start_sha, end_sha, author, message) + gitaly_operation_client.user_squash(user, start_sha, end_sha, author, message) end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index fd794acb4dd..c17934f12c3 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -162,6 +162,14 @@ module Gitlab raise Gitlab::Git::CommitError, 'failed to apply merge to branch' unless branch_update.commit_id.present? Gitlab::Git::OperationService::BranchUpdate.from_gitaly(branch_update) + + rescue GRPC::BadStatus => e + decoded_error = decode_detailed_error(e) + + raise unless decoded_error.present? + + raise decoded_error + ensure request_enum.close end @@ -259,11 +267,10 @@ module Gitlab request_enum.close end - def user_squash(user, squash_id, start_sha, end_sha, author, message, time = Time.now.utc) + def user_squash(user, start_sha, end_sha, author, message, time = Time.now.utc) request = Gitaly::UserSquashRequest.new( repository: @gitaly_repo, user: Gitlab::Git::User.from_gitlab(user).to_gitaly, - squash_id: squash_id.to_s, start_sha: start_sha, end_sha: end_sha, author: Gitlab::Git::User.from_gitlab(author).to_gitaly, @@ -471,6 +478,31 @@ module Gitlab rescue RangeError raise ArgumentError, "Unknown action '#{action[:action]}'" end + + def decode_detailed_error(err) + # details could have more than one in theory, but we only have one to worry about for now. + detailed_error = err.to_rpc_status&.details&.first + + return unless detailed_error.present? + + prefix = %r{type\.googleapis\.com\/gitaly\.(?<error_type>.+)} + error_type = prefix.match(detailed_error.type_url)[:error_type] + + detailed_error = Gitaly.const_get(error_type, false).decode(detailed_error.value) + + case detailed_error.error + when :access_check + access_check_error = detailed_error.access_check + # These messages were returned from internal/allowed API calls + Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) + else + # We're handling access_check only for now, but we'll add more detailed error types + nil + end + rescue NameError, NoMethodError + # Error Class might not be known to ruby yet + nil + end end end end diff --git a/lib/gitlab/github_import/parallel_importer.rb b/lib/gitlab/github_import/parallel_importer.rb index 2429fa4de1d..f72e595e8e9 100644 --- a/lib/gitlab/github_import/parallel_importer.rb +++ b/lib/gitlab/github_import/parallel_importer.rb @@ -15,6 +15,10 @@ module Gitlab true end + def self.track_start_import(project) + Gitlab::Import::Metrics.new(:github_importer, project).track_start_import + end + # This is a workaround for a Ruby 2.3.7 bug. rspec-mocks cannot restore # the visibility of prepended modules. See # https://github.com/rspec/rspec-mocks/issues/1231 for more details. diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 4d0074e43d7..a8e006ea082 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -53,7 +53,8 @@ module Gitlab project_id: project.id, error_source: self.class.name, exception: e, - fail_import: abort_on_failure + fail_import: abort_on_failure, + metrics: true ) raise(e) diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index d0584cc6255..a3dcd2e380c 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -11,7 +11,7 @@ module Gitlab expose_attribute :noteable_type, :noteable_id, :commit_id, :file_path, :diff_hunk, :author, :note, :created_at, :updated_at, - :github_id, :original_commit_id + :original_commit_id, :note_id NOTEABLE_ID_REGEX = %r{/pull/(?<iid>\d+)}i.freeze @@ -40,7 +40,9 @@ module Gitlab note: note.body, created_at: note.created_at, updated_at: note.updated_at, - github_id: note.id + note_id: note.id, + end_line: note.line, + start_line: note.start_line } new(hash) @@ -82,6 +84,22 @@ module Gitlab new_file: false } end + + def note + @note ||= DiffNotes::SuggestionFormatter.formatted_note_for( + note: attributes[:note], + start_line: attributes[:start_line], + end_line: attributes[:end_line] + ) + end + + def github_identifiers + { + note_id: note_id, + noteable_id: noteable_id, + noteable_type: noteable_type + } + end end end end diff --git a/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb b/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb new file mode 100644 index 00000000000..4e5855ee4cd --- /dev/null +++ b/lib/gitlab/github_import/representation/diff_notes/suggestion_formatter.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +# This class replaces Github markdown suggestion tag with +# a Gitlab suggestion tag. The difference between +# Github's and Gitlab's suggestion tags is that Gitlab +# includes the range of the suggestion in the tag, while Github +# uses other note attributes to position the suggestion. +module Gitlab + module GithubImport + module Representation + module DiffNotes + class SuggestionFormatter + # A github suggestion: + # - the ```suggestion tag must be the first text of the line + # - it might have up to 3 spaces before the ```suggestion tag + # - extra text on the ```suggestion tag line will be ignored + GITHUB_SUGGESTION = /^\ {,3}(?<suggestion>```suggestion\b).*(?<eol>\R)/.freeze + + def self.formatted_note_for(...) + new(...).formatted_note + end + + def initialize(note:, start_line: nil, end_line: nil) + @note = note + @start_line = start_line + @end_line = end_line + end + + def formatted_note + if contains_suggestion? + note.gsub( + GITHUB_SUGGESTION, + "\\k<suggestion>:#{suggestion_range}\\k<eol>" + ) + else + note + end + end + + private + + attr_reader :note, :start_line, :end_line + + def contains_suggestion? + note.to_s.match?(GITHUB_SUGGESTION) + end + + # Github always saves the comment on the _last_ line of the range. + # Therefore, the diff hunk will always be related to lines before + # the comment itself. + def suggestion_range + "-#{line_count}+0" + end + + def line_count + if start_line.present? + end_line - start_line + else + 0 + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb index 0e04b5ad57f..db4a8188c03 100644 --- a/lib/gitlab/github_import/representation/issue.rb +++ b/lib/gitlab/github_import/representation/issue.rb @@ -25,7 +25,6 @@ module Gitlab hash = { iid: issue.number, - github_id: issue.number, title: issue.title, description: issue.body, milestone_number: issue.milestone&.number, @@ -75,6 +74,13 @@ module Gitlab def issuable_type pull_request? ? 'MergeRequest' : 'Issue' end + + def github_identifiers + { + iid: iid, + issuable_type: issuable_type + } + end end end end diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb index 41723759645..18737bfcde3 100644 --- a/lib/gitlab/github_import/representation/lfs_object.rb +++ b/lib/gitlab/github_import/representation/lfs_object.rb @@ -16,8 +16,7 @@ module Gitlab new( oid: lfs_object.oid, link: lfs_object.link, - size: lfs_object.size, - github_id: lfs_object.oid + size: lfs_object.size ) end @@ -31,6 +30,12 @@ module Gitlab def initialize(attributes) @attributes = attributes end + + def github_identifiers + { + oid: oid + } + end end end end diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index 5b98ce7d5ed..bcdb1a5459b 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :attributes expose_attribute :noteable_id, :noteable_type, :author, :note, - :created_at, :updated_at, :github_id + :created_at, :updated_at, :note_id NOTEABLE_TYPE_REGEX = %r{/(?<type>(pull|issues))/(?<iid>\d+)}i.freeze @@ -42,7 +42,7 @@ module Gitlab note: note.body, created_at: note.created_at, updated_at: note.updated_at, - github_id: note.id + note_id: note.id } new(hash) @@ -64,6 +64,14 @@ module Gitlab end alias_method :issuable_type, :noteable_type + + def github_identifiers + { + note_id: note_id, + noteable_id: noteable_id, + noteable_type: noteable_type + } + end end end end diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index e4f54fcc833..82bcdee8b2b 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -25,7 +25,6 @@ module Gitlab hash = { iid: pr.number, - github_id: pr.number, title: pr.title, description: pr.body, source_branch: pr.head.ref, @@ -108,6 +107,13 @@ module Gitlab def issuable_type 'MergeRequest' end + + def github_identifiers + { + iid: iid, + issuable_type: issuable_type + } + end end end end diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb index 08b3160fc4c..70c1e51ffdd 100644 --- a/lib/gitlab/github_import/representation/pull_request_review.rb +++ b/lib/gitlab/github_import/representation/pull_request_review.rb @@ -9,7 +9,7 @@ module Gitlab attr_reader :attributes - expose_attribute :author, :note, :review_type, :submitted_at, :github_id, :merge_request_id + expose_attribute :author, :note, :review_type, :submitted_at, :merge_request_id, :review_id def self.from_api_response(review) user = Representation::User.from_api_response(review.user) if review.user @@ -20,7 +20,7 @@ module Gitlab note: review.body, review_type: review.state, submitted_at: review.submitted_at, - github_id: review.id + review_id: review.id ) end @@ -43,6 +43,13 @@ module Gitlab def approval? review_type == 'APPROVED' end + + def github_identifiers + { + review_id: review_id, + merge_request_id: merge_request_id + } + end end end end diff --git a/lib/gitlab/github_import/representation/user.rb b/lib/gitlab/github_import/representation/user.rb index d97b90b6291..fac8920a3f2 100644 --- a/lib/gitlab/github_import/representation/user.rb +++ b/lib/gitlab/github_import/representation/user.rb @@ -17,7 +17,6 @@ module Gitlab def self.from_api_response(user) new( id: user.id, - github_id: user.id, login: user.login ) end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb index cb6b2017208..6bc37337799 100644 --- a/lib/gitlab/github_import/sequential_importer.rb +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -33,18 +33,41 @@ module Gitlab end def execute - Importer::RepositoryImporter.new(project, client).execute + metrics.track_start_import - SEQUENTIAL_IMPORTERS.each do |klass| - klass.new(project, client).execute + begin + Importer::RepositoryImporter.new(project, client).execute + + SEQUENTIAL_IMPORTERS.each do |klass| + klass.new(project, client).execute + end + + rescue StandardError => e + Gitlab::Import::ImportFailureService.track( + project_id: project.id, + error_source: self.class.name, + exception: e, + fail_import: true, + metrics: true + ) + + raise(e) end PARALLEL_IMPORTERS.each do |klass| klass.new(project, client, parallel: false).execute end + metrics.track_finished_import + true end + + private + + def metrics + @metrics ||= Gitlab::Import::Metrics.new(:github_importer, project) + end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 258c13894fb..9f628a10771 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -55,6 +55,7 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix, default_enabled: false) push_frontend_feature_flag(:improved_emoji_picker, default_enabled: :yaml) push_frontend_feature_flag(:new_header_search, default_enabled: :yaml) + push_frontend_feature_flag(:suppress_apollo_errors_during_navigation, current_user, default_enabled: :yaml) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/grape_logging/loggers/context_logger.rb b/lib/gitlab/grape_logging/loggers/context_logger.rb index 468a296886e..1da96fdfdff 100644 --- a/lib/gitlab/grape_logging/loggers/context_logger.rb +++ b/lib/gitlab/grape_logging/loggers/context_logger.rb @@ -1,11 +1,18 @@ # frozen_string_literal: true -# This module adds additional correlation id the grape logger +# This class adds application context to the grape logger module Gitlab module GrapeLogging module Loggers class ContextLogger < ::GrapeLogging::Loggers::Base - def parameters(_, _) + def parameters(request, _) + # Add remote_ip if this request wasn't already handled. If we + # add it unconditionally we can break client_id due to the way + # the context inherits the user. + unless Gitlab::ApplicationContext.current_context_include?(:remote_ip) + Gitlab::ApplicationContext.push(remote_ip: request.ip) + end + Gitlab::ApplicationContext.current end end diff --git a/lib/gitlab/graphql/board/issues_connection_extension.rb b/lib/gitlab/graphql/board/issues_connection_extension.rb new file mode 100644 index 00000000000..9dcd8c92592 --- /dev/null +++ b/lib/gitlab/graphql/board/issues_connection_extension.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true +module Gitlab + module Graphql + module Board + class IssuesConnectionExtension < GraphQL::Schema::Field::ConnectionExtension + def after_resolve(value:, object:, context:, **rest) + ::Boards::Issues::ListService + .initialize_relative_positions(object.list.board, context[:current_user], value.nodes) + + value + end + end + end + end +end diff --git a/lib/gitlab/graphql/connection_collection_methods.rb b/lib/gitlab/graphql/connection_collection_methods.rb index 0e2c4a98bb6..2818a9d4e88 100644 --- a/lib/gitlab/graphql/connection_collection_methods.rb +++ b/lib/gitlab/graphql/connection_collection_methods.rb @@ -6,7 +6,7 @@ module Gitlab extend ActiveSupport::Concern included do - delegate :to_a, :size, :include?, :empty?, to: :nodes + delegate :to_a, :size, :map, :include?, :empty?, to: :nodes end end end diff --git a/lib/gitlab/health_checks/redis/rate_limiting_check.rb b/lib/gitlab/health_checks/redis/rate_limiting_check.rb new file mode 100644 index 00000000000..67c14e26361 --- /dev/null +++ b/lib/gitlab/health_checks/redis/rate_limiting_check.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + module Redis + class RateLimitingCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_rate_limiting_ping' + end + + def successful?(result) + result == 'PONG' + end + + # rubocop: disable CodeReuse/ActiveRecord + def check + catch_timeout 10.seconds do + Gitlab::Redis::RateLimiting.with(&:ping) + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb index 44b85bf886e..25879c18f84 100644 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ b/lib/gitlab/health_checks/redis/redis_check.rb @@ -21,7 +21,9 @@ module Gitlab ::Gitlab::HealthChecks::Redis::CacheCheck.check_up && ::Gitlab::HealthChecks::Redis::QueuesCheck.check_up && ::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up && - ::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up + ::Gitlab::HealthChecks::Redis::TraceChunksCheck.check_up && + ::Gitlab::HealthChecks::Redis::RateLimitingCheck.check_up && + ::Gitlab::HealthChecks::Redis::SessionsCheck.check_up end end end diff --git a/lib/gitlab/health_checks/redis/sessions_check.rb b/lib/gitlab/health_checks/redis/sessions_check.rb new file mode 100644 index 00000000000..a0c5e177b4e --- /dev/null +++ b/lib/gitlab/health_checks/redis/sessions_check.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + module Redis + class SessionsCheck + extend SimpleAbstractCheck + + class << self + def check_up + check + end + + private + + def metric_prefix + 'redis_sessions_ping' + end + + def successful?(result) + result == 'PONG' + end + + # rubocop: disable CodeReuse/ActiveRecord + def check + catch_timeout 10.seconds do + Gitlab::Redis::Sessions.with(&:ping) + end + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index f830af68e07..49712548960 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -70,7 +70,7 @@ module Gitlab end def highlight_plain(text) - @formatter.format(Rouge::Lexers::PlainText.lex(text), context).html_safe + @formatter.format(Rouge::Lexers::PlainText.lex(text), **context).html_safe end def highlight_rich(text, continue: true) @@ -78,7 +78,7 @@ module Gitlab tag = lexer.tag tokens = lexer.lex(text, continue: continue) - Timeout.timeout(timeout_time) { @formatter.format(tokens, context.merge(tag: tag)).html_safe } + Timeout.timeout(timeout_time) { @formatter.format(tokens, **context, tag: tag).html_safe } rescue Timeout::Error => e add_highlight_timeout_metric diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 33f2916345e..b090d05de19 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -43,27 +43,27 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 25, + 'da_DK' => 52, 'de' => 16, 'en' => 100, 'eo' => 0, - 'es' => 42, + 'es' => 41, 'fil_PH' => 0, 'fr' => 11, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 2, - 'ja' => 38, - 'ko' => 12, - 'nb_NO' => 26, + 'ja' => 37, + 'ko' => 11, + 'nb_NO' => 35, 'nl_NL' => 0, - 'pl_PL' => 6, + 'pl_PL' => 5, 'pt_BR' => 45, - 'ro_RO' => 21, - 'ru' => 28, + 'ro_RO' => 24, + 'ru' => 27, 'tr_TR' => 16, 'uk' => 40, - 'zh_CN' => 94, + 'zh_CN' => 95, 'zh_HK' => 2, 'zh_TW' => 3 }.freeze diff --git a/lib/gitlab/import/import_failure_service.rb b/lib/gitlab/import/import_failure_service.rb index f808ed1b6e2..142c00f7a6b 100644 --- a/lib/gitlab/import/import_failure_service.rb +++ b/lib/gitlab/import/import_failure_service.rb @@ -8,14 +8,15 @@ module Gitlab import_state: nil, project_id: nil, error_source: nil, - fail_import: false + fail_import: false, + metrics: false ) new( exception: exception, import_state: import_state, project_id: project_id, error_source: error_source - ).execute(fail_import: fail_import) + ).execute(fail_import: fail_import, metrics: metrics) end def initialize(exception:, import_state: nil, project_id: nil, error_source: nil) @@ -35,10 +36,11 @@ module Gitlab @error_source = error_source end - def execute(fail_import:) + def execute(fail_import:, metrics:) track_exception persist_failure + track_metrics if metrics import_state.mark_as_failed(exception.message) if fail_import end @@ -71,6 +73,10 @@ module Gitlab correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id ) end + + def track_metrics + Gitlab::Import::Metrics.new("#{project.import_type}_importer", project).track_failed_import + end end end end diff --git a/lib/gitlab/import/metrics.rb b/lib/gitlab/import/metrics.rb index 2692ab2fa12..5f27d0ab965 100644 --- a/lib/gitlab/import/metrics.rb +++ b/lib/gitlab/import/metrics.rb @@ -3,27 +3,35 @@ module Gitlab module Import class Metrics + include Gitlab::Utils::UsageData + IMPORT_DURATION_BUCKETS = [0.5, 1, 3, 5, 10, 60, 120, 240, 360, 720, 1440].freeze - attr_reader :importer + attr_reader :importer, :duration def initialize(importer, project) @importer = importer @project = project end + def track_start_import + return unless project.github_import? + + track_usage_event(:github_import_project_start, project.id) + end + def track_finished_import - duration = Time.zone.now - @project.created_at + @duration = Time.zone.now - project.created_at - duration_histogram.observe({ importer: importer }, duration) + observe_histogram projects_counter.increment + track_finish_metric end - def projects_counter - @projects_counter ||= Gitlab::Metrics.counter( - :"#{importer}_imported_projects_total", - 'The number of imported projects' - ) + def track_failed_import + return unless project.github_import? + + track_usage_event(:github_import_project_failure, project.id) end def issues_counter @@ -42,6 +50,8 @@ module Gitlab private + attr_reader :project + def duration_histogram @duration_histogram ||= Gitlab::Metrics.histogram( :"#{importer}_total_duration_seconds", @@ -50,6 +60,27 @@ module Gitlab IMPORT_DURATION_BUCKETS ) end + + def projects_counter + @projects_counter ||= Gitlab::Metrics.counter( + :"#{importer}_imported_projects_total", + 'The number of imported projects' + ) + end + + def observe_histogram + if project.github_import? + duration_histogram.observe({ project: project.full_path }, duration) + else + duration_histogram.observe({ importer: importer }, duration) + end + end + + def track_finish_metric + return unless project.github_import? + + track_usage_event(:github_import_project_success, project.id) + end end end end diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb index acd03d9ec20..2d8e25a9f70 100644 --- a/lib/gitlab/import_export/attributes_permitter.rb +++ b/lib/gitlab/import_export/attributes_permitter.rb @@ -44,7 +44,7 @@ module Gitlab # We want to use AttributesCleaner for these relations instead, in the future this should be removed to make sure # we are using AttributesPermitter for every imported relation. - DISABLED_RELATION_NAMES = %i[user author ci_cd_settings issuable_sla push_rule].freeze + DISABLED_RELATION_NAMES = %i[user author issuable_sla].freeze def initialize(config: ImportExport::Config.new.to_h) @config = config diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index a84efd1d240..6749ef4e276 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -183,7 +183,7 @@ module Gitlab def parsed_relation_hash strong_memoize(:parsed_relation_hash) do - if Feature.enabled?(:permitted_attributes_for_import_export, default_enabled: :yaml) && attributes_permitter.permitted_attributes_defined?(@relation_sym) + if use_attributes_permitter? && attributes_permitter.permitted_attributes_defined?(@relation_sym) attributes_permitter.permit(@relation_sym, @relation_hash) else Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, relation_class: relation_class) @@ -195,6 +195,10 @@ module Gitlab @attributes_permitter ||= Gitlab::ImportExport::AttributesPermitter.new end + def use_attributes_permitter? + Feature.enabled?(:permitted_attributes_for_import_export, default_enabled: :yaml) + end + def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 6c0b6de9e85..fdc4c22001f 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -14,6 +14,10 @@ module Gitlab untar_with_options(archive: archive, dir: dir, options: 'zxf') end + def tar_cf(archive:, dir:) + tar_with_options(archive: archive, dir: dir, options: 'cf') + end + def gzip(dir:, filename:) gzip_with_options(dir: dir, filename: filename) end @@ -59,19 +63,29 @@ module Gitlab end def tar_with_options(archive:, dir:, options:) - execute(%W(tar -#{options} #{archive} -C #{dir} .)) + execute_cmd(%W(tar -#{options} #{archive} -C #{dir} .)) end def untar_with_options(archive:, dir:, options:) - execute(%W(tar -#{options} #{archive} -C #{dir})) - execute(%W(chmod -R #{UNTAR_MASK} #{dir})) + execute_cmd(%W(tar -#{options} #{archive} -C #{dir})) + execute_cmd(%W(chmod -R #{UNTAR_MASK} #{dir})) end - def execute(cmd) + # rubocop:disable Gitlab/ModuleWithInstanceVariables + def execute_cmd(cmd) output, status = Gitlab::Popen.popen(cmd) - @shared.error(Gitlab::ImportExport::Error.new(output.to_s)) unless status == 0 # rubocop:disable Gitlab/ModuleWithInstanceVariables - status == 0 + + return true if status == 0 + + if @shared.respond_to?(:error) + @shared.error(Gitlab::ImportExport::Error.new(output.to_s)) + + false + else + raise Gitlab::ImportExport::Error, 'System call failed' + end end + # rubocop:enable Gitlab/ModuleWithInstanceVariables def git_bin_path Gitlab.config.git.bin_path diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb index 91637161377..adbbd37ce10 100644 --- a/lib/gitlab/import_export/group/relation_factory.rb +++ b/lib/gitlab/import_export/group/relation_factory.rb @@ -36,6 +36,10 @@ module Gitlab @relation_hash['group_id'] = @importable.id end + + def use_attributes_permitter? + false + end end end end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 9d28e1abeab..9ab8fa68d0e 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -171,7 +171,6 @@ module Gitlab def read_from_replica_if_available(&block) return yield unless ::Feature.enabled?(:load_balancing_for_export_workers, type: :development, default_enabled: :yaml) - return yield unless ::Gitlab::Database::LoadBalancing.enable? ::Gitlab::Database::LoadBalancing::Session.current.use_replicas_for_read_queries(&block) end diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index 3910afef108..301e90e3171 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -39,7 +39,9 @@ module Gitlab # created manually. Ignore failures so we get the merge request itself if # the commits are missing. def create_source_branch - @project.repository.create_branch(@merge_request.source_branch, @diff_head_sha) + if @merge_request.open? + @project.repository.create_branch(@merge_request.source_branch, @diff_head_sha) + end rescue StandardError => err Gitlab::Import::Logger.warn( message: 'Import warning: Failed to create source branch', diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 8046fedc4f3..86fd11cc336 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -131,7 +131,6 @@ included_attributes: - :link_url - :name - :project_id - - :type - :updated_at pipeline_schedules: - :active @@ -155,6 +154,124 @@ included_attributes: - :enabled - :project_id - :updated_at + boards: + - :project_id + - :created_at + - :updated_at + - :group_id + - :weight + - :name + - :hide_backlog_list + - :hide_closed_list + lists: + - :list_type + - :position + - :created_at + - :updated_at + - :user_id + - :max_issue_count + - :max_issue_weight + - :limit_metric + custom_attributes: + - :created_at + - :updated_at + - :project_id + - :key + - :value + label: + - :title + - :color + - :project_id + - :group_id + - :created_at + - :updated_at + - :template + - :description + - :priority + labels: + - :title + - :color + - :project_id + - :group_id + - :created_at + - :updated_at + - :template + - :description + - :priority + priorities: + - :project_id + - :priority + - :created_at + - :updated_at + milestone: + - :iid + - :title + - :project_id + - :group_id + - :description + - :due_date + - :created_at + - :updated_at + - :start_date + - :state + milestones: + - :iid + - :title + - :project_id + - :group_id + - :description + - :due_date + - :created_at + - :updated_at + - :start_date + - :state + protected_branches: + - :project_id + - :name + - :created_at + - :updated_at + - :code_owner_approval_required + - :allow_force_push + protected_tags: + - :project_id + - :name + - :created_at + - :updated_at + create_access_levels: + - :access_level + - :created_at + - :updated_at + - :user_id + - :group_id + merge_access_levels: + - :access_level + - :created_at + - :updated_at + - :user_id + - :group_id + push_access_levels: + - :access_level + - :created_at + - :updated_at + - :user_id + - :group_id + releases: + - :tag + - :description + - :project_id + - :author_id + - :created_at + - :updated_at + - :name + - :sha + - :released_at + links: + - :url + - :name + - :created_at + - :updated_at + - :filepath + - :link_type # Do not include the following attributes for the models specified. excluded_attributes: @@ -498,6 +615,10 @@ ee: - :deploy_access_levels - :security_setting - :push_rule + - boards: + - :milestone + - lists: + - :milestone included_attributes: issuable_sla: @@ -519,3 +640,20 @@ ee: - :reject_unsigned_commits - :commit_committer_check - :regexp_uses_re2 + unprotect_access_levels: + - :access_level + - :user_id + - :group_id + deploy_access_levels: + - :created_at + - :updated_at + - :access_level + - :user_id + - :group_id + protected_environments: + - :project_id + - :group_id + - :name + - :created_at + - :updated_at + diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 8d93098a80a..1eeacafef53 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -37,7 +37,7 @@ module Gitlab ActiveRecord::Base.no_touching do update_params! - BulkInsertableAssociations.with_bulk_insert(enabled: @importable.instance_of?(::Project)) do + BulkInsertableAssociations.with_bulk_insert(enabled: project?) do fix_ci_pipelines_not_sorted_on_legacy_project_json! create_relations! end @@ -55,6 +55,10 @@ module Gitlab private + def project? + @importable.instance_of?(::Project) + end + # Loops through the tree of models defined in import_export.yml and # finds them in the imported JSON so they can be instantiated and saved # in the DB. The structure and relationships between models are guessed from @@ -75,7 +79,7 @@ module Gitlab def process_relation_item!(relation_key, relation_definition, relation_index, data_hash) relation_object = build_relation(relation_key, relation_definition, relation_index, data_hash) return unless relation_object - return if importable_class == ::Project && group_model?(relation_object) + return if project? && group_model?(relation_object) relation_object.assign_attributes(importable_class_sym => @importable) @@ -114,7 +118,8 @@ module Gitlab excluded_keys: excluded_keys_for_relation(importable_class_sym)) @importable.assign_attributes(params) - @importable.drop_visibility_level! if importable_class == ::Project + + modify_attributes Gitlab::Timeless.timeless(@importable) do @importable.save! @@ -141,6 +146,13 @@ module Gitlab end end + def modify_attributes + return unless project? + + @importable.reconcile_shared_runners_setting! + @importable.drop_visibility_level! + end + def build_relations(relation_key, relation_definition, relation_index, data_hashes) data_hashes .map { |data_hash| build_relation(relation_key, relation_definition, relation_index, data_hash) } diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index ab0e56adc32..4fee779c767 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -9,8 +9,10 @@ module Gitlab Queues = Class.new(RedisBase) SharedState = Class.new(RedisBase).enable_redis_cluster_validation TraceChunks = Class.new(RedisBase).enable_redis_cluster_validation + RateLimiting = Class.new(RedisBase).enable_redis_cluster_validation + Sessions = Class.new(RedisBase).enable_redis_cluster_validation - STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks].freeze + STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks, RateLimiting, Sessions].freeze # Milliseconds represented in seconds (from 1 millisecond to 2 seconds). QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 23acf1e8e86..26e44d7822e 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -131,18 +131,43 @@ module Gitlab enqueued_at_time = convert_to_time(enqueued_at) return unless enqueued_at_time - # Its possible that if theres clock-skew between two nodes - # this value may be less than zero. In that event, we record the value + round_elapsed_time(enqueued_at_time) + end + + # Returns the time it took for a scheduled job to be enqueued in seconds, as a float, + # if the `scheduled_at` and `enqueued_at` fields are available. + # + # * If the job doesn't contain sufficient information, returns nil + # * If the job has a start time in the future, returns 0 + # * If the job contains an invalid start time value, returns nil + # @param [Hash] job a Sidekiq job, represented as a hash + def self.enqueue_latency_for_scheduled_job(job) + scheduled_at = job['scheduled_at'] + enqueued_at = job['enqueued_at'] + + return unless scheduled_at && enqueued_at + + scheduled_at_time = convert_to_time(scheduled_at) + enqueued_at_time = convert_to_time(enqueued_at) + + return unless scheduled_at_time && enqueued_at_time + + round_elapsed_time(scheduled_at_time, enqueued_at_time) + end + + def self.round_elapsed_time(start, end_time = Time.now) + # It's possible that if there is clock-skew between two nodes this + # value may be less than zero. In that event, we record the value # as zero. - [elapsed_by_absolute_time(enqueued_at_time), 0].max.round(DURATION_PRECISION) + [elapsed_by_absolute_time(start, end_time), 0].max.round(DURATION_PRECISION) end # Calculates the time in seconds, as a float, from # the provided start time until now # # @param [Time] start - def self.elapsed_by_absolute_time(start) - (Time.now - start).to_f.round(DURATION_PRECISION) + def self.elapsed_by_absolute_time(start, end_time) + (end_time - start).to_f.round(DURATION_PRECISION) end private_class_method :elapsed_by_absolute_time diff --git a/lib/gitlab/issuable_sorter.rb b/lib/gitlab/issuable_sorter.rb index 42bbfb32d0b..45c7dc295b1 100644 --- a/lib/gitlab/issuable_sorter.rb +++ b/lib/gitlab/issuable_sorter.rb @@ -7,7 +7,7 @@ module Gitlab grouped_items = issuables.group_by do |issuable| if issuable.project.id == project.id :project_ref - elsif issuable.project.namespace.id == project.namespace.id + elsif issuable.project.namespace_id == project.namespace_id :namespace_ref else :full_ref diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index 45582f19214..408b3afc128 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -41,6 +41,10 @@ module Gitlab end def tunnel_url + configured = Gitlab.config.gitlab_kas['external_k8s_proxy_url'] + return configured if configured.present? + + # Legacy code path. Will be removed when all distributions provide a sane default here uri = URI.join(external_url, K8S_PROXY_PATH) uri.scheme = uri.scheme.in?(%w(grpcs wss)) ? 'https' : 'http' uri.to_s diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 0633efc6b0c..75d27ed8cc1 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -71,7 +71,8 @@ module Gitlab def redis_config gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env) - config = { redis_url: gitlab_redis_queues.url } + + config = { redis_url: gitlab_redis_queues.url, redis_db: gitlab_redis_queues.db } if gitlab_redis_queues.sentinels? config[:sentinels] = gitlab_redis_queues.sentinels diff --git a/lib/gitlab/merge_requests/mergeability/check_result.rb b/lib/gitlab/merge_requests/mergeability/check_result.rb new file mode 100644 index 00000000000..d0788c7d7c7 --- /dev/null +++ b/lib/gitlab/merge_requests/mergeability/check_result.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true +module Gitlab + module MergeRequests + module Mergeability + class CheckResult + SUCCESS_STATUS = :success + FAILED_STATUS = :failed + + attr_reader :status, :payload + + def self.default_payload + { last_run_at: Time.current } + end + + def self.success(payload: {}) + new(status: SUCCESS_STATUS, payload: default_payload.merge(payload)) + end + + def self.failed(payload: {}) + new(status: FAILED_STATUS, payload: default_payload.merge(payload)) + end + + def self.from_hash(data) + new( + status: data.fetch(:status), + payload: data.fetch(:payload)) + end + + def initialize(status:, payload: {}) + @status = status + @payload = payload + end + + def to_hash + { status: status, payload: payload } + end + + def failed? + status == FAILED_STATUS + end + + def success? + status == SUCCESS_STATUS + end + end + end + end +end diff --git a/lib/gitlab/merge_requests/mergeability/redis_interface.rb b/lib/gitlab/merge_requests/mergeability/redis_interface.rb new file mode 100644 index 00000000000..081ccfca360 --- /dev/null +++ b/lib/gitlab/merge_requests/mergeability/redis_interface.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +module Gitlab + module MergeRequests + module Mergeability + class RedisInterface + EXPIRATION = 6.hours + VERSION = 1 + + def save_check(merge_check:, result_hash:) + Gitlab::Redis::SharedState.with do |redis| + redis.set(merge_check.cache_key + ":#{VERSION}", result_hash.to_json, ex: EXPIRATION) + end + end + + def retrieve_check(merge_check:) + Gitlab::Redis::SharedState.with do |redis| + Gitlab::Json.parse(redis.get(merge_check.cache_key + ":#{VERSION}")) + end + end + end + end + end +end diff --git a/lib/gitlab/merge_requests/mergeability/results_store.rb b/lib/gitlab/merge_requests/mergeability/results_store.rb new file mode 100644 index 00000000000..bb6489f8526 --- /dev/null +++ b/lib/gitlab/merge_requests/mergeability/results_store.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true +module Gitlab + module MergeRequests + module Mergeability + class ResultsStore + def initialize(interface: RedisInterface.new, merge_request:) + @interface = interface + @merge_request = merge_request + end + + def read(merge_check:) + interface.retrieve_check(merge_check: merge_check) + end + + def write(merge_check:, result_hash:) + interface.save_check(merge_check: merge_check, result_hash: result_hash) + end + + private + + attr_reader :interface + end + end + end +end diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb index 641c0c76f8f..6d4b49676e5 100644 --- a/lib/gitlab/metrics/dashboard/service_selector.rb +++ b/lib/gitlab/metrics/dashboard/service_selector.rb @@ -30,7 +30,7 @@ module Gitlab # Returns a class which inherits from the BaseService # class that can be used to obtain a dashboard for # the provided params. - # @return [Gitlab::Metrics::Dashboard::Services::BaseService] + # @return [Metrics::Dashboard::BaseService] def call(params) service = services.find do |service_class| service_class.valid_params?(params) diff --git a/lib/gitlab/metrics/exporter/web_exporter.rb b/lib/gitlab/metrics/exporter/web_exporter.rb index f378577f08e..c5fa1e545d7 100644 --- a/lib/gitlab/metrics/exporter/web_exporter.rb +++ b/lib/gitlab/metrics/exporter/web_exporter.rb @@ -15,6 +15,14 @@ module Gitlab end end + RailsMetricsInitializer = Struct.new(:app) do + def call(env) + Gitlab::Metrics::RailsSlis.initialize_request_slis_if_needed! + + app.call(env) + end + end + attr_reader :running # This exporter is always run on master process @@ -45,6 +53,15 @@ module Gitlab private + def rack_app + app = super + + Rack::Builder.app do + use RailsMetricsInitializer + run app + end + end + def start_working @running = true super diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb deleted file mode 100644 index ad45a037161..00000000000 --- a/lib/gitlab/metrics/instrumentation.rb +++ /dev/null @@ -1,194 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Metrics - # Module for instrumenting methods. - # - # This module allows instrumenting of methods without having to actually - # alter the target code (e.g. by including modules). - # - # Example usage: - # - # Gitlab::Metrics::Instrumentation.instrument_method(User, :by_login) - module Instrumentation - PROXY_IVAR = :@__gitlab_instrumentation_proxy - - def self.configure - yield self - end - - # Returns the name of the series to use for storing method calls. - def self.series - @series ||= "#{::Gitlab::Metrics.series_prefix}method_calls" - end - - # Instruments a class method. - # - # mod - The module to instrument as a Module/Class. - # name - The name of the method to instrument. - def self.instrument_method(mod, name) - instrument(:class, mod, name) - end - - # Instruments an instance method. - # - # mod - The module to instrument as a Module/Class. - # name - The name of the method to instrument. - def self.instrument_instance_method(mod, name) - instrument(:instance, mod, name) - end - - # Recursively instruments all subclasses of the given root module. - # - # This can be used to for example instrument all ActiveRecord models (as - # these all inherit from ActiveRecord::Base). - # - # This method can optionally take a block to pass to `instrument_methods` - # and `instrument_instance_methods`. - # - # root - The root module for which to instrument subclasses. The root - # module itself is not instrumented. - def self.instrument_class_hierarchy(root, &block) - visit = root.subclasses - - until visit.empty? - klass = visit.pop - - instrument_methods(klass, &block) - instrument_instance_methods(klass, &block) - - klass.subclasses.each { |c| visit << c } - end - end - - # Instruments all public and private methods of a module. - # - # This method optionally takes a block that can be used to determine if a - # method should be instrumented or not. The block is passed the receiving - # module and an UnboundMethod. If the block returns a non truthy value the - # method is not instrumented. - # - # mod - The module to instrument. - def self.instrument_methods(mod) - methods = mod.methods(false) + mod.private_methods(false) - methods.each do |name| - method = mod.method(name) - - if method.owner == mod.singleton_class - if !block_given? || block_given? && yield(mod, method) - instrument_method(mod, name) - end - end - end - end - - # Instruments all public and private instance methods of a module. - # - # See `instrument_methods` for more information. - # - # mod - The module to instrument. - def self.instrument_instance_methods(mod) - methods = mod.instance_methods(false) + mod.private_instance_methods(false) - methods.each do |name| - method = mod.instance_method(name) - - if method.owner == mod - if !block_given? || block_given? && yield(mod, method) - instrument_instance_method(mod, name) - end - end - end - end - - # Returns true if a module is instrumented. - # - # mod - The module to check - def self.instrumented?(mod) - mod.instance_variable_defined?(PROXY_IVAR) - end - - # Returns the proxy module (if any) of `mod`. - def self.proxy_module(mod) - mod.instance_variable_get(PROXY_IVAR) - end - - # Instruments a method. - # - # type - The type (:class or :instance) of method to instrument. - # mod - The module containing the method. - # name - The name of the method to instrument. - def self.instrument(type, mod, name) - return unless ::Gitlab::Metrics.enabled? - - if type == :instance - target = mod - method_name = "##{name}" - method = mod.instance_method(name) - else - target = mod.singleton_class - method_name = ".#{name}" - method = mod.method(name) - end - - label = "#{mod.name}#{method_name}" - - unless instrumented?(target) - target.instance_variable_set(PROXY_IVAR, Module.new) - end - - proxy_module = self.proxy_module(target) - - # Some code out there (e.g. the "state_machine" Gem) checks the arity of - # a method to make sure it only passes arguments when the method expects - # any. If we were to always overwrite a method to take an `*args` - # signature this would break things. As a result we'll make sure the - # generated method _only_ accepts regular arguments if the underlying - # method also accepts them. - args_signature = - if method.arity == 0 - '' - else - '*args' - end - - method_visibility = method_visibility_for(target, name) - - # We silence warnings to avoid such warnings: - # `Skipping set of ruby2_keywords flag for <...> - # (method accepts keywords or method does not accept argument splat)` - # as we apply ruby2_keywords 'blindly' for every instrumented method. - proxy_module.class_eval <<-EOF, __FILE__, __LINE__ + 1 - def #{name}(#{args_signature}) - if trans = Gitlab::Metrics::Instrumentation.transaction - trans.method_call_for(#{label.to_sym.inspect}, #{mod.name.inspect}, "#{method_name}") - .measure { super } - else - super - end - end - silence_warnings { ruby2_keywords(:#{name}) if respond_to?(:ruby2_keywords, true) } - #{method_visibility} :#{name} - EOF - - target.prepend(proxy_module) - end - - def self.method_visibility_for(mod, name) - if mod.private_method_defined?(name) - :private - elsif mod.protected_method_defined?(name) - :protected - else - :public - end - end - private_class_method :method_visibility_for - - # Small layer of indirection to make it easier to stub out the current - # transaction. - def self.transaction - Transaction.current - end - end - end -end diff --git a/lib/gitlab/metrics/rails_slis.rb b/lib/gitlab/metrics/rails_slis.rb new file mode 100644 index 00000000000..69e0c1e9fde --- /dev/null +++ b/lib/gitlab/metrics/rails_slis.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module RailsSlis + class << self + def request_apdex_counters_enabled? + Feature.enabled?(:request_apdex_counters) + end + + def initialize_request_slis_if_needed! + return unless request_apdex_counters_enabled? + return if Gitlab::Metrics::Sli.initialized?(:rails_request_apdex) + + Gitlab::Metrics::Sli.initialize_sli(:rails_request_apdex, possible_request_labels) + end + + def request_apdex + Gitlab::Metrics::Sli[:rails_request_apdex] + end + + private + + def possible_request_labels + possible_controller_labels + possible_api_labels + end + + def possible_api_labels + Gitlab::RequestEndpoints.all_api_endpoints.map do |route| + endpoint_id = API::Base.endpoint_id_for_route(route) + route_class = route.app.options[:for] + feature_category = route_class.feature_category_for_app(route.app) + + { + endpoint_id: endpoint_id, + feature_category: feature_category + } + end + end + + def possible_controller_labels + Gitlab::RequestEndpoints.all_controller_actions.map do |controller, action| + { + endpoint_id: controller.endpoint_id_for_action(action), + feature_category: controller.feature_category_for_action(action) + } + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index 6ba336d37cd..3a0e34d5615 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -15,7 +15,8 @@ module Gitlab HEALTH_ENDPOINT = %r{^/-/(liveness|readiness|health|metrics)/?$}.freeze - FEATURE_CATEGORY_DEFAULT = 'unknown' + FEATURE_CATEGORY_DEFAULT = ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT + ENDPOINT_MISSING = 'unknown' # These were the top 5 categories at a point in time, chosen as a # reasonable default. If we initialize every category we'll end up @@ -77,6 +78,8 @@ module Gitlab if !health_endpoint && ::Gitlab::Metrics.record_duration_for_status?(status) self.class.http_request_duration_seconds.observe({ method: method }, elapsed) + + record_apdex_if_needed(env, elapsed) end [status, headers, body] @@ -105,6 +108,39 @@ module Gitlab def feature_category ::Gitlab::ApplicationContext.current_context_attribute(:feature_category) end + + def endpoint_id + ::Gitlab::ApplicationContext.current_context_attribute(:caller_id) + end + + def record_apdex_if_needed(env, elapsed) + return unless Gitlab::Metrics::RailsSlis.request_apdex_counters_enabled? + + Gitlab::Metrics::RailsSlis.request_apdex.increment( + labels: labels_from_context, + success: satisfactory?(env, elapsed) + ) + end + + def labels_from_context + { + feature_category: feature_category.presence || FEATURE_CATEGORY_DEFAULT, + endpoint_id: endpoint_id.presence || ENDPOINT_MISSING + } + end + + def satisfactory?(env, elapsed) + target = + if env['api.endpoint'].present? + env['api.endpoint'].options[:for].try(:urgency_for_app, env['api.endpoint']) + elsif env['action_controller.instance'].present? && env['action_controller.instance'].respond_to?(:urgency) + env['action_controller.instance'].urgency + end + + target ||= Gitlab::EndpointAttributes::DEFAULT_URGENCY + + elapsed < target.duration + end end end end diff --git a/lib/gitlab/metrics/sli.rb b/lib/gitlab/metrics/sli.rb new file mode 100644 index 00000000000..de73db0755d --- /dev/null +++ b/lib/gitlab/metrics/sli.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + class Sli + SliNotInitializedError = Class.new(StandardError) + + COUNTER_PREFIX = 'gitlab_sli' + + class << self + INITIALIZATION_MUTEX = Mutex.new + + def [](name) + known_slis[name] || initialize_sli(name, []) + end + + def initialize_sli(name, possible_label_combinations) + INITIALIZATION_MUTEX.synchronize do + sli = new(name) + sli.initialize_counters(possible_label_combinations) + known_slis[name] = sli + end + end + + def initialized?(name) + known_slis.key?(name) && known_slis[name].initialized? + end + + private + + def known_slis + @known_slis ||= {} + end + end + + attr_reader :name + + def initialize(name) + @name = name + @initialized_with_combinations = false + end + + def initialize_counters(possible_label_combinations) + @initialized_with_combinations = possible_label_combinations.any? + possible_label_combinations.each do |label_combination| + total_counter.get(label_combination) + success_counter.get(label_combination) + end + end + + def increment(labels:, success:) + total_counter.increment(labels) + success_counter.increment(labels) if success + end + + def initialized? + @initialized_with_combinations + end + + private + + def total_counter + prometheus.counter(total_counter_name.to_sym, "Total number of measurements for #{name}") + end + + def success_counter + prometheus.counter(success_counter_name.to_sym, "Number of successful measurements for #{name}") + end + + def total_counter_name + "#{COUNTER_PREFIX}:#{name}:total" + end + + def success_counter_name + "#{COUNTER_PREFIX}:#{name}:success_total" + end + + def prometheus + Gitlab::Metrics + end + end + end +end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index 59b2f88041f..df0582149a9 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -47,13 +47,11 @@ module Gitlab buckets SQL_DURATION_BUCKET end - if ::Gitlab::Database::LoadBalancing.enable? - db_role = ::Gitlab::Database::LoadBalancing.db_role_for_connection(payload[:connection]) - return if db_role.blank? + db_role = ::Gitlab::Database::LoadBalancing.db_role_for_connection(payload[:connection]) + return if db_role.blank? - increment_db_role_counters(db_role, payload) - observe_db_role_duration(db_role, event) - end + increment_db_role_counters(db_role, payload) + observe_db_role_duration(db_role, event) end def self.db_counter_payload @@ -64,7 +62,7 @@ module Gitlab payload[key] = Gitlab::SafeRequestStore[key].to_i end - if ::Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + if ::Gitlab::SafeRequestStore.active? load_balancing_metric_counter_keys.each do |counter| payload[counter] = ::Gitlab::SafeRequestStore[counter].to_i end diff --git a/lib/gitlab/metrics/subscribers/load_balancing.rb b/lib/gitlab/metrics/subscribers/load_balancing.rb index 333fc63ebef..bd77e8c3c3f 100644 --- a/lib/gitlab/metrics/subscribers/load_balancing.rb +++ b/lib/gitlab/metrics/subscribers/load_balancing.rb @@ -10,7 +10,7 @@ module Gitlab LOG_COUNTERS = { true => :caught_up_replica_pick_ok, false => :caught_up_replica_pick_fail }.freeze def caught_up_replica_pick(event) - return unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + return unless Gitlab::SafeRequestStore.active? result = event.payload[:result] counter_name = counter(result) @@ -20,13 +20,13 @@ module Gitlab # we want to update Prometheus counter after the controller/action are set def web_transaction_completed(_event) - return unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + return unless Gitlab::SafeRequestStore.active? LOG_COUNTERS.keys.each { |result| increment_prometheus_for_result_label(result) } end def self.load_balancing_payload - return {} unless Gitlab::SafeRequestStore.active? && ::Gitlab::Database::LoadBalancing.enable? + return {} unless Gitlab::SafeRequestStore.active? {}.tap do |payload| LOG_COUNTERS.values.each do |counter| diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb index ebd0d1634e7..d86c0f83c6c 100644 --- a/lib/gitlab/metrics/subscribers/rack_attack.rb +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -22,7 +22,8 @@ module Gitlab :throttle_authenticated_protected_paths_web, :throttle_authenticated_packages_api, :throttle_authenticated_git_lfs, - :throttle_authenticated_files_api + :throttle_authenticated_files_api, + :throttle_authenticated_deprecated_api ].freeze PAYLOAD_KEYS = [ diff --git a/lib/gitlab/metrics/web_transaction.rb b/lib/gitlab/metrics/web_transaction.rb index 3ebfcc43b0b..544c142f7bb 100644 --- a/lib/gitlab/metrics/web_transaction.rb +++ b/lib/gitlab/metrics/web_transaction.rb @@ -57,10 +57,6 @@ module Gitlab action = "#{controller.action_name}" - # Try to get the feature category, but don't fail when the controller is - # not an ApplicationController. - feature_category = controller.class.try(:feature_category_for_action, action).to_s - # Devise exposes a method called "request_format" that does the below. # However, this method is not available to all controllers (e.g. certain # Doorkeeper controllers). As such we use the underlying code directly. @@ -91,9 +87,6 @@ module Gitlab if route path = endpoint_paths_cache[route.request_method][route.path] - grape_class = endpoint.options[:for] - feature_category = grape_class.try(:feature_category_for_app, endpoint).to_s - { controller: 'Grape', action: "#{route.request_method} #{path}", feature_category: feature_category } end end @@ -109,6 +102,10 @@ module Gitlab def endpoint_instrumentable_path(raw_path) raw_path.sub('(.:format)', '').sub('/:version', '') end + + def feature_category + ::Gitlab::ApplicationContext.current_context_attribute(:feature_category) || ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT + end end end end diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index 49be3ffc839..a047015e54f 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -158,6 +158,7 @@ module Gitlab ::Gitlab.config.uploads.storage_path, ::JobArtifactUploader.workhorse_upload_path, ::LfsObjectUploader.workhorse_upload_path, + ::DependencyProxy::FileUploader.workhorse_upload_path, File.join(Rails.root, 'public/uploads/tmp') ] + package_allowed_paths end diff --git a/lib/gitlab/middleware/speedscope.rb b/lib/gitlab/middleware/speedscope.rb index 74f334d9ab3..6992ac9b720 100644 --- a/lib/gitlab/middleware/speedscope.rb +++ b/lib/gitlab/middleware/speedscope.rb @@ -19,11 +19,12 @@ module Gitlab require 'stackprof' begin + mode = stackprof_mode(request) flamegraph = ::StackProf.run( - mode: :wall, + mode: mode, raw: true, aggregate: false, - interval: ::Gitlab::StackProf::DEFAULT_INTERVAL_US + interval: ::Gitlab::StackProf.interval(mode) ) do _, _, body = @app.call(env) end @@ -64,7 +65,7 @@ module Gitlab var iframe = document.createElement('IFRAME'); iframe.setAttribute('id', 'speedscope-iframe'); document.body.appendChild(iframe); - var iframeUrl = '#{speedscope_path}#profileURL=' + objUrl + '&title=' + 'Flamegraph for #{CGI.escape(path)}'; + var iframeUrl = '#{speedscope_path}#profileURL=' + objUrl + '&title=' + 'Flamegraph for #{CGI.escape(path)} in #{stackprof_mode(request)} mode'; iframe.setAttribute('src', iframeUrl); </script> </body> @@ -73,6 +74,17 @@ module Gitlab [200, headers, [html]] end + + def stackprof_mode(request) + case request.params['stackprof_mode']&.to_sym + when :cpu + :cpu + when :object + :object + else + :wall + end + end end end end diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index b5e304599ab..9f39b5f122f 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -11,7 +11,7 @@ module Gitlab retry_attempts = 0 begin - ActiveRecord::Base.transaction do # rubocop: disable Database/MultipleDatabases + subject.transaction do yield(subject) end rescue ActiveRecord::StaleObjectError diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb index 39d6e016ac7..53faf8469f2 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb @@ -6,10 +6,7 @@ module Gitlab module InOperatorOptimization # rubocop: disable CodeReuse/ActiveRecord class QueryBuilder - UnsupportedScopeOrder = Class.new(StandardError) - RECURSIVE_CTE_NAME = 'recursive_keyset_cte' - RECORDS_COLUMN = 'records' # This class optimizes slow database queries (PostgreSQL specific) where the # IN SQL operator is used with sorting. @@ -42,26 +39,19 @@ module Gitlab # > array_mapping_scope: array_mapping_scope, # > finder_query: finder_query # > ).execute.limit(20) - def initialize(scope:, array_scope:, array_mapping_scope:, finder_query:, values: {}) + def initialize(scope:, array_scope:, array_mapping_scope:, finder_query: nil, values: {}) @scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) - unless success - error_message = <<~MSG - The order on the scope does not support keyset pagination. You might need to define a custom Order object.\n - See https://docs.gitlab.com/ee/development/database/keyset_pagination.html#complex-order-configuration\n - Or the Gitlab::Pagination::Keyset::Order class for examples - MSG - raise(UnsupportedScopeOrder, error_message) - end + raise(UnsupportedScopeOrder) unless success @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) @array_scope = array_scope @array_mapping_scope = array_mapping_scope - @finder_query = finder_query @values = values @model = @scope.model @table_name = @model.table_name @arel_table = @model.arel_table + @finder_strategy = finder_query.present? ? Strategies::RecordLoaderStrategy.new(finder_query, model, order_by_columns) : Strategies::OrderValuesLoaderStrategy.new(model, order_by_columns) end def execute @@ -74,7 +64,7 @@ module Gitlab q = cte .apply_to(model.where({}) .with(selector_cte.to_arel)) - .select(result_collector_final_projections) + .select(finder_strategy.final_projections) .where("count <> 0") # filter out the initializer row model.from(q.arel.as(table_name)) @@ -82,13 +72,13 @@ module Gitlab private - attr_reader :array_scope, :scope, :order, :array_mapping_scope, :finder_query, :values, :model, :table_name, :arel_table + attr_reader :array_scope, :scope, :order, :array_mapping_scope, :finder_strategy, :values, :model, :table_name, :arel_table def initializer_query array_column_names = array_scope_columns.array_aggregated_column_names + order_by_columns.array_aggregated_column_names projections = [ - *result_collector_initializer_columns, + *finder_strategy.initializer_columns, *array_column_names, '0::bigint AS count' ] @@ -156,7 +146,7 @@ module Gitlab order_column_value_arrays = order_by_columns.replace_value_in_array_by_position_expressions select = [ - *result_collector_columns, + *finder_strategy.columns, *array_column_list, *order_column_value_arrays, "#{RECURSIVE_CTE_NAME}.count + 1" @@ -254,23 +244,6 @@ module Gitlab end.join(", ") end - def result_collector_initializer_columns - ["NULL::#{table_name} AS #{RECORDS_COLUMN}"] - end - - def result_collector_columns - query = finder_query - .call(*order_by_columns.array_lookup_expressions_by_position(RECURSIVE_CTE_NAME)) - .select("#{table_name}") - .limit(1) - - ["(#{query.to_sql})"] - end - - def result_collector_final_projections - ["(#{RECORDS_COLUMN}).*"] - end - def array_scope_columns @array_scope_columns ||= ArrayScopeColumns.new(array_scope.select_values) end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb new file mode 100644 index 00000000000..fc2b56048f6 --- /dev/null +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/order_values_loader_strategy.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + module InOperatorOptimization + module Strategies + class OrderValuesLoaderStrategy + def initialize(model, order_by_columns) + @model = model + @order_by_columns = order_by_columns + end + + def initializer_columns + order_by_columns.map do |column| + column_name = column.original_column_name.to_s + type = model.columns_hash[column_name].sql_type + "NULL::#{type} AS #{column_name}" + end + end + + def columns + order_by_columns.array_lookup_expressions_by_position(QueryBuilder::RECURSIVE_CTE_NAME) + end + + def final_projections + order_by_columns.map(&:original_column_name) + end + + private + + attr_reader :model, :order_by_columns + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb new file mode 100644 index 00000000000..b12c33d6e51 --- /dev/null +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + module InOperatorOptimization + module Strategies + class RecordLoaderStrategy + RECORDS_COLUMN = 'records' + + def initialize(finder_query, model, order_by_columns) + @finder_query = finder_query + @order_by_columns = order_by_columns + @table_name = model.table_name + end + + def initializer_columns + ["NULL::#{table_name} AS #{RECORDS_COLUMN}"] + end + + def columns + query = finder_query + .call(*order_by_columns.array_lookup_expressions_by_position(QueryBuilder::RECURSIVE_CTE_NAME)) + .select("#{table_name}") + .limit(1) + + ["(#{query.to_sql})"] + end + + def final_projections + ["(#{RECORDS_COLUMN}).*"] + end + + private + + attr_reader :finder_query, :order_by_columns, :table_name + end + end + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/iterator.rb b/lib/gitlab/pagination/keyset/iterator.rb index 14807fa37c4..bcd17fd0d34 100644 --- a/lib/gitlab/pagination/keyset/iterator.rb +++ b/lib/gitlab/pagination/keyset/iterator.rb @@ -4,12 +4,11 @@ module Gitlab module Pagination module Keyset class Iterator - UnsupportedScopeOrder = Class.new(StandardError) - - def initialize(scope:, use_union_optimization: true, in_operator_optimization_options: nil) + def initialize(scope:, cursor: {}, use_union_optimization: true, in_operator_optimization_options: nil) @scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) - raise(UnsupportedScopeOrder, 'The order on the scope does not support keyset pagination') unless success + raise(UnsupportedScopeOrder) unless success + @cursor = cursor @order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) @use_union_optimization = in_operator_optimization_options ? false : use_union_optimization @in_operator_optimization_options = in_operator_optimization_options @@ -17,11 +16,9 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def each_batch(of: 1000) - cursor_attributes = {} - loop do current_scope = scope.dup - relation = order.apply_cursor_conditions(current_scope, cursor_attributes, keyset_options) + relation = order.apply_cursor_conditions(current_scope, cursor, keyset_options) relation = relation.reorder(order) unless @in_operator_optimization_options relation = relation.limit(of) @@ -30,14 +27,14 @@ module Gitlab last_record = relation.last break unless last_record - cursor_attributes = order.cursor_attributes_for_node(last_record) + @cursor = order.cursor_attributes_for_node(last_record) end end # rubocop: enable CodeReuse/ActiveRecord private - attr_reader :scope, :order + attr_reader :scope, :cursor, :order def keyset_options { diff --git a/lib/gitlab/pagination/keyset/paginator.rb b/lib/gitlab/pagination/keyset/paginator.rb index 1c71549d86a..1ff4589d8e1 100644 --- a/lib/gitlab/pagination/keyset/paginator.rb +++ b/lib/gitlab/pagination/keyset/paginator.rb @@ -19,8 +19,6 @@ module Gitlab FORWARD_DIRECTION = 'n' BACKWARD_DIRECTION = 'p' - UnsupportedScopeOrder = Class.new(StandardError) - # scope - ActiveRecord::Relation object with order by clause # cursor - Encoded cursor attributes as String. Empty value will requests the first page. # per_page - Number of items per page. @@ -167,7 +165,7 @@ module Gitlab def build_scope(scope) keyset_aware_scope, success = Gitlab::Pagination::Keyset::SimpleOrderBuilder.build(scope) - raise(UnsupportedScopeOrder, 'The order on the scope does not support keyset pagination') unless success + raise(UnsupportedScopeOrder) unless success keyset_aware_scope end diff --git a/lib/gitlab/pagination/keyset/unsupported_scope_order.rb b/lib/gitlab/pagination/keyset/unsupported_scope_order.rb new file mode 100644 index 00000000000..1571c00e130 --- /dev/null +++ b/lib/gitlab/pagination/keyset/unsupported_scope_order.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Pagination + module Keyset + class UnsupportedScopeOrder < StandardError + DEFAULT_ERROR_MESSAGE = <<~MSG + The order on the scope does not support keyset pagination. You might need to define a custom Order object.\n + See https://docs.gitlab.com/ee/development/database/keyset_pagination.html#complex-order-configuration\n + Or the Gitlab::Pagination::Keyset::Order class for examples + MSG + + def message + DEFAULT_ERROR_MESSAGE + end + end + end + end +end diff --git a/lib/gitlab/path_regex.rb b/lib/gitlab/path_regex.rb index c648f4d1fd0..06a26c4830f 100644 --- a/lib/gitlab/path_regex.rb +++ b/lib/gitlab/path_regex.rb @@ -262,6 +262,10 @@ module Gitlab @container_image_blob_sha_regex ||= %r{[\w+.-]+:?\w+}.freeze end + def dependency_proxy_route_regex + @dependency_proxy_route_regex ||= %r{\A/v2/#{full_namespace_route_regex}/dependency_proxy/containers/#{container_image_regex}/(manifests|blobs)/#{container_image_blob_sha_regex}\z} + end + private def personal_snippet_path_regex diff --git a/lib/gitlab/performance_bar/stats.rb b/lib/gitlab/performance_bar/stats.rb index 103cd65cb4b..cf524e69454 100644 --- a/lib/gitlab/performance_bar/stats.rb +++ b/lib/gitlab/performance_bar/stats.rb @@ -9,6 +9,9 @@ module Gitlab ee/lib/ee/peek lib/peek lib/gitlab/database + lib/gitlab/gitaly_client.rb + lib/gitlab/gitaly_client/call.rb + lib/gitlab/instrumentation/redis_interceptor.rb ].freeze def initialize(redis) @@ -19,7 +22,9 @@ module Gitlab data = request(id) return unless data - log_sql_queries(id, data) + log_queries(id, data, 'active-record') + log_queries(id, data, 'gitaly') + log_queries(id, data, 'redis') rescue StandardError => err logger.error(message: "failed to process request id #{id}: #{err.message}") end @@ -32,15 +37,15 @@ module Gitlab Gitlab::Json.parse(json_data) end - def log_sql_queries(id, data) - queries_by_location(data).each do |location, queries| + def log_queries(id, data, type) + queries_by_location(data, type).each do |location, queries| next unless location duration = queries.sum { |query| query['duration'].to_f } log_info = { method_path: "#{location[:filename]}:#{location[:method]}", filename: location[:filename], - type: :sql, + query_type: type, request_id: id, count: queries.count, duration_ms: duration @@ -50,8 +55,8 @@ module Gitlab end end - def queries_by_location(data) - return [] unless queries = data.dig('data', 'active-record', 'details') + def queries_by_location(data, type) + return [] unless queries = data.dig('data', type, 'details') queries.group_by do |query| parse_backtrace(query['backtrace']) diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 6348a4902f8..cc2021e14e3 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -148,7 +148,7 @@ module Gitlab quick_action_target.persisted? && quick_action_target.can_be_approved_by?(current_user) end command :approve do - success = MergeRequests::ApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) + success = ::MergeRequests::ApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) next unless success @@ -162,7 +162,7 @@ module Gitlab quick_action_target.persisted? && quick_action_target.can_be_unapproved_by?(current_user) end command :unapprove do - success = MergeRequests::RemoveApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) + success = ::MergeRequests::RemoveApprovalService.new(project: quick_action_target.project, current_user: current_user).execute(quick_action_target) next unless success @@ -275,7 +275,7 @@ module Gitlab end def merge_orchestration_service - @merge_orchestration_service ||= MergeRequests::MergeOrchestrationService.new(project, current_user) + @merge_orchestration_service ||= ::MergeRequests::MergeOrchestrationService.new(project, current_user) end def preferred_auto_merge_strategy(merge_request) diff --git a/lib/gitlab/quick_actions/relate_actions.rb b/lib/gitlab/quick_actions/relate_actions.rb index 95f71214667..1de23523f01 100644 --- a/lib/gitlab/quick_actions/relate_actions.rb +++ b/lib/gitlab/quick_actions/relate_actions.rb @@ -17,11 +17,17 @@ module Gitlab params '#issue' types Issue condition do - quick_action_target.persisted? && - current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) + current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) end - command :relate do |related_param| - IssueLinks::CreateService.new(quick_action_target, current_user, { issuable_references: [related_param] }).execute + command :relate do |related_reference| + service = IssueLinks::CreateService.new(quick_action_target, current_user, { issuable_references: [related_reference] }) + create_issue_link = proc { service.execute } + + if quick_action_target.persisted? + create_issue_link.call + else + quick_action_target.run_after_commit(&create_issue_link) + end end end end diff --git a/lib/gitlab/rack_attack/instrumented_cache_store.rb b/lib/gitlab/rack_attack/instrumented_cache_store.rb index 8cf9082384f..d8beb259fba 100644 --- a/lib/gitlab/rack_attack/instrumented_cache_store.rb +++ b/lib/gitlab/rack_attack/instrumented_cache_store.rb @@ -2,9 +2,10 @@ module Gitlab module RackAttack - # This class is a proxy for all Redis calls made by RackAttack. All the - # calls are instrumented, then redirected to ::Rails.cache. This class - # instruments the standard interfaces of ActiveRecord::Cache defined in + # This class is a proxy for all Redis calls made by RackAttack. All + # the calls are instrumented, then redirected to the underlying + # store (in `.store). This class instruments the standard interfaces + # of ActiveRecord::Cache defined in # https://github.com/rails/rails/blob/v6.0.3.1/activesupport/lib/active_support/cache.rb#L315 # # For more information, please see @@ -14,7 +15,7 @@ module Gitlab delegate :silence!, :mute, to: :@upstream_store - def initialize(upstream_store: ::Rails.cache, notifier: ActiveSupport::Notifications) + def initialize(upstream_store: ::Gitlab::Redis::RateLimiting.cache_store, notifier: ActiveSupport::Notifications) @upstream_store = upstream_store @notifier = notifier end diff --git a/lib/gitlab/rack_attack/request.rb b/lib/gitlab/rack_attack/request.rb index 099174842d0..dbc77c9f9d7 100644 --- a/lib/gitlab/rack_attack/request.rb +++ b/lib/gitlab/rack_attack/request.rb @@ -4,6 +4,7 @@ module Gitlab module RackAttack module Request FILES_PATH_REGEX = %r{^/api/v\d+/projects/[^/]+/repository/files/.+}.freeze + GROUP_PATH_REGEX = %r{^/api/v\d+/groups/[^/]+/?$}.freeze def unauthenticated? !(authenticated_user_id([:api, :rss, :ics]) || authenticated_runner_id) @@ -71,6 +72,7 @@ module Gitlab !should_be_skipped? && !throttle_unauthenticated_packages_api? && !throttle_unauthenticated_files_api? && + !throttle_unauthenticated_deprecated_api? && Gitlab::Throttle.settings.throttle_unauthenticated_api_enabled && unauthenticated? end @@ -87,6 +89,7 @@ module Gitlab api_request? && !throttle_authenticated_packages_api? && !throttle_authenticated_files_api? && + !throttle_authenticated_deprecated_api? && Gitlab::Throttle.settings.throttle_authenticated_api_enabled end @@ -147,6 +150,17 @@ module Gitlab Gitlab::Throttle.settings.throttle_authenticated_files_api_enabled end + def throttle_unauthenticated_deprecated_api? + deprecated_api_request? && + Gitlab::Throttle.settings.throttle_unauthenticated_deprecated_api_enabled && + unauthenticated? + end + + def throttle_authenticated_deprecated_api? + deprecated_api_request? && + Gitlab::Throttle.settings.throttle_authenticated_deprecated_api_enabled + end + private def authenticated_user_id(request_formats) @@ -176,6 +190,15 @@ module Gitlab def files_api_path? path =~ FILES_PATH_REGEX end + + def deprecated_api_request? + # The projects member of the groups endpoint is deprecated. If left + # unspecified, with_projects defaults to true + with_projects = params['with_projects'] + with_projects = true if with_projects.blank? + + path =~ GROUP_PATH_REGEX && Gitlab::Utils.to_boolean(with_projects) + end end end end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 98b66080b42..a2c7b5e29db 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -5,12 +5,15 @@ module Gitlab class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab' - private - - def raw_config_hash - config = super - config[:url] = 'redis://localhost:6380' if config[:url].blank? - config + # Full list of options: + # https://api.rubyonrails.org/classes/ActiveSupport/Cache/RedisCacheStore.html#method-c-new + def self.active_support_config + { + redis: pool, + compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), + namespace: CACHE_NAMESPACE, + expires_in: 2.weeks # Cache should not grow forever + } end end end diff --git a/lib/gitlab/redis/queues.rb b/lib/gitlab/redis/queues.rb index 9e291a73bb6..e60e59dcf01 100644 --- a/lib/gitlab/redis/queues.rb +++ b/lib/gitlab/redis/queues.rb @@ -2,21 +2,12 @@ # We need this require for MailRoom require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper) -require 'active_support/core_ext/object/blank' module Gitlab module Redis class Queues < ::Gitlab::Redis::Wrapper SIDEKIQ_NAMESPACE = 'resque:gitlab' MAILROOM_NAMESPACE = 'mail_room:gitlab' - - private - - def raw_config_hash - config = super - config[:url] = 'redis://localhost:6381' if config[:url].blank? - config - end end end end diff --git a/lib/gitlab/redis/rate_limiting.rb b/lib/gitlab/redis/rate_limiting.rb new file mode 100644 index 00000000000..4ae1d55e4ce --- /dev/null +++ b/lib/gitlab/redis/rate_limiting.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class RateLimiting < ::Gitlab::Redis::Wrapper + # The data we store on RateLimiting used to be stored on Cache. + def self.config_fallback + Cache + end + + def self.cache_store + @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new(redis: pool, namespace: Cache::CACHE_NAMESPACE) + end + end + end +end diff --git a/lib/gitlab/redis/sessions.rb b/lib/gitlab/redis/sessions.rb new file mode 100644 index 00000000000..3bf1eb6211d --- /dev/null +++ b/lib/gitlab/redis/sessions.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class Sessions < ::Gitlab::Redis::Wrapper + # The data we store on Sessions used to be stored on SharedState. + def self.config_fallback + SharedState + end + end + end +end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index d62516bd287..1250eabb041 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -7,14 +7,6 @@ module Gitlab USER_SESSIONS_NAMESPACE = 'session:user:gitlab' USER_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:user:gitlab' IP_SESSIONS_LOOKUP_NAMESPACE = 'session:lookup:ip:gitlab2' - - private - - def raw_config_hash - config = super - config[:url] = 'redis://localhost:6382' if config[:url].blank? - config - end end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 3c8ac07215d..7b804038146 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -6,6 +6,7 @@ # Rails. require 'active_support/core_ext/hash/keys' require 'active_support/core_ext/module/delegation' +require 'active_support/core_ext/object/blank' require 'active_support/core_ext/string/inflections' # Explicitly load Redis::Store::Factory so we can read Redis configuration in @@ -95,6 +96,8 @@ module Gitlab end def instrumentation_class + return unless defined?(::Gitlab::Instrumentation::Redis) + "::Gitlab::Instrumentation::Redis::#{store_name}".constantize end end @@ -111,6 +114,10 @@ module Gitlab raw_config_hash[:url] end + def db + redis_store_options[:db] + end + def sentinels raw_config_hash[:sentinels] end @@ -150,11 +157,35 @@ module Gitlab def raw_config_hash config_data = fetch_config - if config_data - config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys - else - { url: '' } + config_hash = + if config_data + config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys + else + { url: '' } + end + + if config_hash[:url].blank? + config_hash[:url] = legacy_fallback_urls[self.class.store_name] || legacy_fallback_urls[self.class.config_fallback.store_name] end + + config_hash + end + + # These URLs were defined for cache, queues, and shared_state in + # code. They are used only when no config file exists at all for a + # given instance. The configuration does not seem particularly + # useful - it uses different ports on localhost - but we cannot + # confidently delete it as we don't know if any instances rely on + # this. + # + # DO NOT ADD new instances here. All new instances should define a + # `.config_fallback`, which will then be used to look up this URL. + def legacy_fallback_urls + { + 'Cache' => 'redis://localhost:6380', + 'Queues' => 'redis://localhost:6381', + 'SharedState' => 'redis://localhost:6382' + } end def fetch_config diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index a88ef5fe73e..8b2f786a91a 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -131,7 +131,7 @@ module Gitlab end def helm_channel_regex - @helm_channel_regex ||= %r{\A[-\.\_a-zA-Z0-9]+\z}.freeze + @helm_channel_regex ||= %r{\A([a-zA-Z0-9](\.|-|_)?){1,255}(?<!\.|-|_)\z}.freeze end def helm_package_regex @@ -220,12 +220,12 @@ module Gitlab # The character range \p{Alnum} overlaps with \u{00A9}-\u{1f9ff} # hence the Ruby warning. # https://gitlab.com/gitlab-org/gitlab/merge_requests/23165#not-easy-fixable - @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{00A9}-\u{1f9ff}_\. ]*\z/.freeze + @project_name_regex ||= /\A[\p{Alnum}\u{00A9}-\u{1f9ff}_][\p{Alnum}\p{Pd}\u{002B}\u{00A9}-\u{1f9ff}_\. ]*\z/.freeze end def project_name_regex_message - "can contain only letters, digits, emojis, '_', '.', dash, space. " \ - "It must start with letter, digit, emoji or '_'." + "can contain only letters, digits, emojis, '_', '.', '+', dashes, or spaces. " \ + "It must start with a letter, digit, emoji, or '_'." end def group_name_regex @@ -409,7 +409,7 @@ module Gitlab end def merge_request_draft - /\A(?i)(\[draft\]|\(draft\)|draft:|draft\s\-\s|draft\z)/ + /\A(?i)(\[draft\]|\(draft\)|draft:)/ end def issue diff --git a/lib/gitlab/request_endpoints.rb b/lib/gitlab/request_endpoints.rb new file mode 100644 index 00000000000..157c0f91e65 --- /dev/null +++ b/lib/gitlab/request_endpoints.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module RequestEndpoints + class << self + def all_api_endpoints + # This compile does not do anything if the routes were already built + # but if they weren't, the routes will be drawn and available for the rest of + # application. + API::API.compile! + API::API.routes.select { |route| route.app.options[:for] < API::Base } + end + + def all_controller_actions + # This will return tuples of all controller actions defined in the routes + # Only for controllers inheriting ApplicationController + # Excluding controllers from gems (OAuth, Sidekiq) + Rails.application.routes.routes.filter_map do |route| + route_info = route.required_defaults.presence + next unless route_info + next if route_info[:controller].blank? || route_info[:action].blank? + + controller = constantize_controller(route_info[:controller]) + next unless controller&.include?(::Gitlab::EndpointAttributes) + next if controller == ApplicationController + next if controller == Devise::UnlocksController + + [controller, route_info[:action]] + end + end + + private + + def constantize_controller(name) + "#{name.camelize}Controller".constantize + rescue NameError + nil # some controllers, like the omniauth ones are dynamic + end + end + end +end diff --git a/lib/gitlab/saas.rb b/lib/gitlab/saas.rb index e87c2a0b700..9220ad1be6c 100644 --- a/lib/gitlab/saas.rb +++ b/lib/gitlab/saas.rb @@ -36,6 +36,14 @@ module Gitlab def self.gitlab_com_status_url 'https://status.gitlab.com' end + + def self.about_pricing_url + "https://about.gitlab.com/pricing" + end + + def self.about_pricing_faq_url + "https://about.gitlab.com/gitlab-com/#faq" + end end end diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index bd6b80530c3..5663c51bb7a 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -23,12 +23,12 @@ module Gitlab DEFAULT_WORKERS = { '_' => DummyWorker.new( queue: 'default', - weight: 1, tags: [] + weight: 1, + tags: [] ), 'ActionMailer::MailDeliveryJob' => DummyWorker.new( name: 'ActionMailer::MailDeliveryJob', queue: 'mailers', - feature_category: :issue_tracking, urgency: 'low', weight: 2, tags: [] diff --git a/lib/gitlab/sidekiq_config/dummy_worker.rb b/lib/gitlab/sidekiq_config/dummy_worker.rb index b7f53da8e00..8a2ea1acaab 100644 --- a/lib/gitlab/sidekiq_config/dummy_worker.rb +++ b/lib/gitlab/sidekiq_config/dummy_worker.rb @@ -6,7 +6,6 @@ module Gitlab class DummyWorker ATTRIBUTE_METHODS = { name: :name, - feature_category: :get_feature_category, has_external_dependencies: :worker_has_external_dependencies?, urgency: :get_urgency, resource_boundary: :get_worker_resource_boundary, @@ -27,6 +26,24 @@ module Gitlab nil end + # All dummy workers are unowned; get the feature category from the + # context if available. + def get_feature_category + Gitlab::ApplicationContext.current_context_attribute('meta.feature_category') || :not_owned + end + + def feature_category_not_owned? + true + end + + def get_worker_context + nil + end + + def context_for_arguments(*) + nil + end + ATTRIBUTE_METHODS.each do |attribute, meth| define_method meth do @attributes[attribute] diff --git a/lib/gitlab/sidekiq_enq.rb b/lib/gitlab/sidekiq_enq.rb new file mode 100644 index 00000000000..d8a01ac8ef4 --- /dev/null +++ b/lib/gitlab/sidekiq_enq.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +# This is a copy of https://github.com/mperham/sidekiq/blob/32c55e31659a1e6bd42f98334cca5eef2863de8d/lib/sidekiq/scheduled.rb#L11-L34 +# +# It effectively reverts +# https://github.com/mperham/sidekiq/commit/9b75467b33759888753191413eddbc15c37a219e +# because we observe that the extra ZREMs caused by this change can lead to high +# CPU usage on Redis at peak times: +# https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1179 +# +module Gitlab + class SidekiqEnq + def enqueue_jobs(now = Time.now.to_f.to_s, sorted_sets = Sidekiq::Scheduled::SETS) + # A job's "score" in Redis is the time at which it should be processed. + # Just check Redis for the set of jobs with a timestamp before now. + Sidekiq.redis do |conn| + sorted_sets.each do |sorted_set| + start_time = ::Gitlab::Metrics::System.monotonic_time + jobs = redundant_jobs = 0 + + Sidekiq.logger.info(message: 'Enqueuing scheduled jobs', status: 'start', sorted_set: sorted_set) + + # Get the next item in the queue if it's score (time to execute) is <= now. + # We need to go through the list one at a time to reduce the risk of something + # going wrong between the time jobs are popped from the scheduled queue and when + # they are pushed onto a work queue and losing the jobs. + while (job = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 1]).first) + + # Pop item off the queue and add it to the work queue. If the job can't be popped from + # the queue, it's because another process already popped it so we can move on to the + # next one. + if conn.zrem(sorted_set, job) + jobs += 1 + Sidekiq::Client.push(Sidekiq.load_json(job)) + else + redundant_jobs += 1 + end + end + + end_time = ::Gitlab::Metrics::System.monotonic_time + Sidekiq.logger.info(message: 'Enqueuing scheduled jobs', + status: 'done', + sorted_set: sorted_set, + jobs_count: jobs, + redundant_jobs_count: redundant_jobs, + duration_s: end_time - start_time) + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index 1aebce987fe..3438bc0f3ef 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -5,7 +5,7 @@ require 'active_record/log_subscriber' module Gitlab module SidekiqLogging - class StructuredLogger + class StructuredLogger < Sidekiq::JobLogger include LogsJobs def call(job, queue) @@ -55,6 +55,9 @@ module Gitlab scheduling_latency_s = ::Gitlab::InstrumentationHelper.queue_duration_for_job(payload) payload['scheduling_latency_s'] = scheduling_latency_s if scheduling_latency_s + enqueue_latency_s = ::Gitlab::InstrumentationHelper.enqueue_latency_for_scheduled_job(payload) + payload['enqueue_latency_s'] = enqueue_latency_s if enqueue_latency_s + payload end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index d084e9e9d7e..c97b1632bf8 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -13,6 +13,13 @@ module Gitlab chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Server chain.add ::Gitlab::SidekiqMiddleware::Monitor + # Labkit wraps the job in the `Labkit::Context` resurrected from + # the job-hash. We need properties from the context for + # recording metrics, so this needs to be before + # `::Gitlab::SidekiqMiddleware::ServerMetrics` (if we're using + # that). + chain.add ::Labkit::Middleware::Sidekiq::Server + if metrics chain.add ::Gitlab::SidekiqMiddleware::ServerMetrics @@ -24,7 +31,6 @@ module Gitlab chain.add ::Gitlab::SidekiqMiddleware::RequestStoreMiddleware chain.add ::Gitlab::SidekiqMiddleware::ExtraDoneLogMetadata chain.add ::Gitlab::SidekiqMiddleware::BatchLoader - chain.add ::Labkit::Middleware::Sidekiq::Server chain.add ::Gitlab::SidekiqMiddleware::InstrumentationLogger chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Server chain.add ::Gitlab::SidekiqVersioning::Middleware @@ -33,7 +39,7 @@ module Gitlab # DuplicateJobs::Server should be placed at the bottom, but before the SidekiqServerMiddleware, # so we can compare the latest WAL location against replica chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server - chain.add ::Gitlab::Database::LoadBalancing::SidekiqServerMiddleware if load_balancing_enabled? + chain.add ::Gitlab::Database::LoadBalancing::SidekiqServerMiddleware end end @@ -46,7 +52,7 @@ module Gitlab chain.add ::Labkit::Middleware::Sidekiq::Client # Sidekiq Client Middleware should be placed before DuplicateJobs::Client middleware, # so we can store WAL location before we deduplicate the job. - chain.add ::Gitlab::Database::LoadBalancing::SidekiqClientMiddleware if load_balancing_enabled? + chain.add ::Gitlab::Database::LoadBalancing::SidekiqClientMiddleware chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Client chain.add ::Gitlab::SidekiqStatus::ClientMiddleware chain.add ::Gitlab::SidekiqMiddleware::AdminMode::Client @@ -55,10 +61,5 @@ module Gitlab chain.add ::Gitlab::SidekiqMiddleware::ClientMetrics end end - - def self.load_balancing_enabled? - ::Gitlab::Database::LoadBalancing.enable? - end - private_class_method :load_balancing_enabled? end end diff --git a/lib/gitlab/sidekiq_middleware/client_metrics.rb b/lib/gitlab/sidekiq_middleware/client_metrics.rb index e3cc7b28c41..ef80ed706f3 100644 --- a/lib/gitlab/sidekiq_middleware/client_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/client_metrics.rb @@ -13,9 +13,15 @@ module Gitlab def call(worker_class, job, queue, _redis_pool) # worker_class can either be the string or class of the worker being enqueued. - worker_class = worker_class.safe_constantize if worker_class.respond_to?(:safe_constantize) + worker_class = worker_class.to_s.safe_constantize + labels = create_labels(worker_class, queue, job) - labels[:scheduling] = job.key?('at') ? 'delayed' : 'immediate' + if job.key?('at') + labels[:scheduling] = 'delayed' + job[:scheduled_at] = job['at'] + else + labels[:scheduling] = 'immediate' + end @metrics.fetch(ENQUEUED).increment(labels, 1) diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index aeb58d7c153..e63164efc94 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -64,9 +64,9 @@ module Gitlab Sidekiq.redis do |redis| redis.multi do |multi| - redis.set(idempotency_key, jid, ex: expiry, nx: true) - read_wal_locations = check_existing_wal_locations!(redis, expiry) - read_jid = redis.get(idempotency_key) + multi.set(idempotency_key, jid, ex: expiry, nx: true) + read_wal_locations = check_existing_wal_locations!(multi, expiry) + read_jid = multi.get(idempotency_key) end end @@ -81,9 +81,9 @@ module Gitlab return unless job_wal_locations.present? Sidekiq.redis do |redis| - redis.multi do + redis.multi do |multi| job_wal_locations.each do |connection_name, location| - redis.eval(LUA_SET_WAL_SCRIPT, keys: [wal_location_key(connection_name)], argv: [location, pg_wal_lsn_diff(connection_name).to_i, WAL_LOCATION_TTL]) + multi.eval(LUA_SET_WAL_SCRIPT, keys: [wal_location_key(connection_name)], argv: [location, pg_wal_lsn_diff(connection_name).to_i, WAL_LOCATION_TTL]) end end end @@ -96,9 +96,9 @@ module Gitlab read_wal_locations = {} Sidekiq.redis do |redis| - redis.multi do + redis.multi do |multi| job_wal_locations.keys.each do |connection_name| - read_wal_locations[connection_name] = redis.lindex(wal_location_key(connection_name), 0) + read_wal_locations[connection_name] = multi.lindex(wal_location_key(connection_name), 0) end end end @@ -110,8 +110,8 @@ module Gitlab def delete! Sidekiq.redis do |redis| redis.multi do |multi| - redis.del(idempotency_key) - delete_wal_locations!(redis) + multi.del(idempotency_key) + delete_wal_locations!(multi) end end end @@ -140,13 +140,14 @@ module Gitlab def idempotent? return false unless worker_klass return false unless worker_klass.respond_to?(:idempotent?) + return false unless preserve_wal_location? || !worker_klass.utilizes_load_balancing_capabilities? worker_klass.idempotent? end private - attr_accessor :existing_wal_locations + attr_writer :existing_wal_locations attr_reader :queue_name, :job attr_writer :existing_jid @@ -154,8 +155,33 @@ module Gitlab @worker_klass ||= worker_class_name.to_s.safe_constantize end + def delete_wal_locations!(redis) + job_wal_locations.keys.each do |connection_name| + redis.del(wal_location_key(connection_name)) + redis.del(existing_wal_location_key(connection_name)) + end + end + + def check_existing_wal_locations!(redis, expiry) + read_wal_locations = {} + + job_wal_locations.each do |connection_name, location| + key = existing_wal_location_key(connection_name) + redis.set(key, location, ex: expiry, nx: true) + read_wal_locations[connection_name] = redis.get(key) + end + + read_wal_locations + end + + def job_wal_locations + return {} unless preserve_wal_location? + + job['wal_locations'] || {} + end + def pg_wal_lsn_diff(connection_name) - Gitlab::Database::DATABASES[connection_name].pg_wal_lsn_diff(job_wal_locations[connection_name], existing_wal_locations[connection_name]) + Gitlab::Database.databases[connection_name].pg_wal_lsn_diff(job_wal_locations[connection_name], existing_wal_locations[connection_name]) end def strategy @@ -178,12 +204,6 @@ module Gitlab job['jid'] end - def job_wal_locations - return {} unless preserve_wal_location? - - job['wal_locations'] || {} - end - def existing_wal_location_key(connection_name) "#{idempotency_key}:#{connection_name}:existing_wal_location" end @@ -208,23 +228,8 @@ module Gitlab "#{worker_class_name}:#{Sidekiq.dump_json(arguments)}" end - def delete_wal_locations!(redis) - job_wal_locations.keys.each do |connection_name| - redis.del(wal_location_key(connection_name)) - redis.del(existing_wal_location_key(connection_name)) - end - end - - def check_existing_wal_locations!(redis, expiry) - read_wal_locations = {} - - job_wal_locations.each do |connection_name, location| - key = existing_wal_location_key(connection_name) - redis.set(key, location, ex: expiry, nx: true) - read_wal_locations[connection_name] = redis.get(key) - end - - read_wal_locations + def existing_wal_locations + @existing_wal_locations ||= {} end def preserve_wal_location? diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb index fc58d4f5323..b0da85b74a6 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb @@ -4,11 +4,15 @@ module Gitlab module SidekiqMiddleware module DuplicateJobs module Strategies - module DeduplicatesWhenScheduling + class DeduplicatesWhenScheduling < Base + extend ::Gitlab::Utils::Override + + override :initialize def initialize(duplicate_job) @duplicate_job = duplicate_job end + override :schedule def schedule(job) if deduplicatable_job? && check! && duplicate_job.duplicate? job['duplicate-of'] = duplicate_job.existing_jid @@ -25,6 +29,7 @@ module Gitlab yield end + override :perform def perform(job) update_job_wal_location!(job) end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb index 5164b994267..25f1b8b7c51 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executed.rb @@ -7,11 +7,7 @@ module Gitlab # This strategy takes a lock before scheduling the job in a queue and # removes the lock after the job has executed preventing a new job to be queued # while a job is still executing. - class UntilExecuted < Base - extend ::Gitlab::Utils::Override - - include DeduplicatesWhenScheduling - + class UntilExecuted < DeduplicatesWhenScheduling override :perform def perform(job) super diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb index 1f7e3a4ea30..693e404af73 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/until_executing.rb @@ -7,11 +7,7 @@ module Gitlab # This strategy takes a lock before scheduling the job in a queue and # removes the lock before the job starts allowing a new job to be queued # while a job is still executing. - class UntilExecuting < Base - extend ::Gitlab::Utils::Override - - include DeduplicatesWhenScheduling - + class UntilExecuting < DeduplicatesWhenScheduling override :perform def perform(job) super diff --git a/lib/gitlab/sidekiq_middleware/metrics_helper.rb b/lib/gitlab/sidekiq_middleware/metrics_helper.rb index 66930a34319..207d2d769b2 100644 --- a/lib/gitlab/sidekiq_middleware/metrics_helper.rb +++ b/lib/gitlab/sidekiq_middleware/metrics_helper.rb @@ -3,14 +3,21 @@ module Gitlab module SidekiqMiddleware module MetricsHelper + include ::Gitlab::SidekiqMiddleware::WorkerContext + TRUE_LABEL = "yes" FALSE_LABEL = "no" private def create_labels(worker_class, queue, job) - worker_name = (job['wrapped'].presence || worker_class).to_s - worker = find_worker(worker_name, worker_class) + worker = find_worker(worker_class, job) + + # This should never happen: we should always be able to find a + # worker class for a given Sidekiq job. But if we can't, we + # shouldn't blow up here, because we want to record this in our + # metrics. + worker_name = worker.try(:name) || worker.class.name labels = { queue: queue.to_s, worker: worker_name, @@ -23,9 +30,7 @@ module Gitlab labels[:urgency] = worker.get_urgency.to_s labels[:external_dependencies] = bool_as_label(worker.worker_has_external_dependencies?) - - feature_category = worker.get_feature_category - labels[:feature_category] = feature_category.to_s + labels[:feature_category] = worker.get_feature_category.to_s resource_boundary = worker.get_worker_resource_boundary labels[:boundary] = resource_boundary == :unknown ? "" : resource_boundary.to_s @@ -36,10 +41,6 @@ module Gitlab def bool_as_label(value) value ? TRUE_LABEL : FALSE_LABEL end - - def find_worker(worker_name, worker_class) - Gitlab::SidekiqConfig::DEFAULT_WORKERS.fetch(worker_name, worker_class) - end end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 2d9767e0266..bea98403997 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -53,10 +53,7 @@ module Gitlab def initialize @metrics = self.class.metrics - - if ::Gitlab::Database::LoadBalancing.enable? - @metrics[:sidekiq_load_balancing_count] = ::Gitlab::Metrics.counter(:sidekiq_load_balancing_count, 'Sidekiq jobs with load balancing') - end + @metrics[:sidekiq_load_balancing_count] = ::Gitlab::Metrics.counter(:sidekiq_load_balancing_count, 'Sidekiq jobs with load balancing') end def call(worker, job, queue) @@ -128,8 +125,6 @@ module Gitlab private def with_load_balancing_settings(job) - return unless ::Gitlab::Database::LoadBalancing.enable? - keys = %w[load_balancing_strategy worker_data_consistency] return unless keys.all? { |k| job.key?(k) } diff --git a/lib/gitlab/sidekiq_middleware/worker_context.rb b/lib/gitlab/sidekiq_middleware/worker_context.rb index 897a9211948..a5d92cf699c 100644 --- a/lib/gitlab/sidekiq_middleware/worker_context.rb +++ b/lib/gitlab/sidekiq_middleware/worker_context.rb @@ -10,6 +10,12 @@ module Gitlab context_or_nil.use(&block) end + + def find_worker(worker_class, job) + worker_name = (job['wrapped'].presence || worker_class).to_s + + Gitlab::SidekiqConfig::DEFAULT_WORKERS[worker_name]&.klass || worker_class + end end end end diff --git a/lib/gitlab/sidekiq_middleware/worker_context/client.rb b/lib/gitlab/sidekiq_middleware/worker_context/client.rb index 1a899b27ea3..7d3925e9dec 100644 --- a/lib/gitlab/sidekiq_middleware/worker_context/client.rb +++ b/lib/gitlab/sidekiq_middleware/worker_context/client.rb @@ -7,11 +7,11 @@ module Gitlab include Gitlab::SidekiqMiddleware::WorkerContext def call(worker_class_or_name, job, _queue, _redis_pool, &block) - worker_class = worker_class_or_name.to_s.safe_constantize + worker_class = find_worker(worker_class_or_name.to_s.safe_constantize, job) - # Mailers can't be constantized like this + # This is not a worker we know about, perhaps from a gem return yield unless worker_class - return yield unless worker_class.include?(::ApplicationWorker) + return yield unless worker_class.respond_to?(:context_for_arguments) context_for_args = worker_class.context_for_arguments(job['args']) @@ -19,7 +19,14 @@ module Gitlab # This should be inside the context for the arguments so # that we don't override the feature category on the worker # with the one from the caller. - Gitlab::ApplicationContext.with_context(feature_category: worker_class.get_feature_category.to_s, &block) + # + # We do not want to set anything explicitly in the context + # when the feature category is 'not_owned'. + if worker_class.feature_category_not_owned? + yield + else + Gitlab::ApplicationContext.with_context(feature_category: worker_class.get_feature_category.to_s, &block) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/worker_context/server.rb b/lib/gitlab/sidekiq_middleware/worker_context/server.rb index 2d8fd8002d2..d026f4918c6 100644 --- a/lib/gitlab/sidekiq_middleware/worker_context/server.rb +++ b/lib/gitlab/sidekiq_middleware/worker_context/server.rb @@ -7,7 +7,7 @@ module Gitlab include Gitlab::SidekiqMiddleware::WorkerContext def call(worker, job, _queue, &block) - worker_class = worker.class + worker_class = find_worker(worker.class, job) # This is not a worker we know about, perhaps from a gem return yield unless worker_class.respond_to?(:get_worker_context) diff --git a/lib/gitlab/sidekiq_versioning.rb b/lib/gitlab/sidekiq_versioning.rb index 8164a5a9d7a..80c0b7650f3 100644 --- a/lib/gitlab/sidekiq_versioning.rb +++ b/lib/gitlab/sidekiq_versioning.rb @@ -3,25 +3,21 @@ module Gitlab module SidekiqVersioning def self.install! - Sidekiq::Manager.prepend SidekiqVersioning::Manager - # The Sidekiq client API always adds the queue to the Sidekiq queue # list, but mail_room and gitlab-shell do not. This is only necessary # for monitoring. - begin - queues = SidekiqConfig.worker_queues + queues = SidekiqConfig.worker_queues - if queues.any? - Sidekiq.redis do |conn| - conn.pipelined do - queues.each do |queue| - conn.sadd('queues', queue) - end + if queues.any? + Sidekiq.redis do |conn| + conn.pipelined do + queues.each do |queue| + conn.sadd('queues', queue) end end end - rescue ::Redis::BaseError, SocketError, Errno::ENOENT, Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED end + rescue ::Redis::BaseError, SocketError, Errno::ENOENT, Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED end end end diff --git a/lib/gitlab/sidekiq_versioning/manager.rb b/lib/gitlab/sidekiq_versioning/manager.rb deleted file mode 100644 index e5852b43003..00000000000 --- a/lib/gitlab/sidekiq_versioning/manager.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SidekiqVersioning - module Manager - def initialize(options = {}) - options[:strict] = false - options[:queues] = SidekiqConfig.expand_queues(options[:queues]) - Sidekiq.logger.info "Listening on queues #{options[:queues].uniq.sort}" - super - end - end - end -end diff --git a/lib/gitlab/stack_prof.rb b/lib/gitlab/stack_prof.rb index 97f52491e9e..9fc4798ffdc 100644 --- a/lib/gitlab/stack_prof.rb +++ b/lib/gitlab/stack_prof.rb @@ -75,20 +75,20 @@ module Gitlab current_timeout_s = nil else mode = ENV['STACKPROF_MODE']&.to_sym || DEFAULT_MODE - interval = ENV['STACKPROF_INTERVAL']&.to_i - interval ||= (mode == :object ? DEFAULT_INTERVAL_EVENTS : DEFAULT_INTERVAL_US) + stackprof_interval = ENV['STACKPROF_INTERVAL']&.to_i + stackprof_interval ||= interval(mode) log_event( 'starting profile', profile_mode: mode, - profile_interval: interval, + profile_interval: stackprof_interval, profile_timeout: timeout_s ) ::StackProf.start( mode: mode, raw: Gitlab::Utils.to_boolean(ENV['STACKPROF_RAW'] || 'true'), - interval: interval + interval: stackprof_interval ) current_timeout_s = timeout_s end @@ -131,5 +131,9 @@ module Gitlab pid: Process.pid }.merge(labels.compact)) end + + def self.interval(mode) + mode == :object ? DEFAULT_INTERVAL_EVENTS : DEFAULT_INTERVAL_US + end end end diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index 78fa5009bc4..9b6bae12057 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -3,7 +3,15 @@ module Gitlab module SubscriptionPortal def self.default_subscriptions_url - ::Gitlab.dev_or_test_env? ? 'https://customers.stg.gitlab.com' : 'https://customers.gitlab.com' + if ::Gitlab.dev_or_test_env? + if Feature.enabled?(:new_customersdot_staging_url, default_enabled: :yaml) + 'https://customers.staging.gitlab.com' + else + 'https://customers.stg.gitlab.com' + end + else + 'https://customers.gitlab.com' + end end def self.subscriptions_url @@ -38,6 +46,26 @@ module Gitlab "#{self.subscriptions_url}/plans" end + def self.subscriptions_gitlab_plans_url + "#{self.subscriptions_url}/gitlab_plans" + end + + def self.subscriptions_instance_review_url + "#{self.subscriptions_url}/instance_review" + end + + def self.add_extra_seats_url(group_id) + "#{self.subscriptions_url}/gitlab/namespaces/#{group_id}/extra_seats" + end + + def self.upgrade_subscription_url(group_id, plan_id) + "#{self.subscriptions_url}/gitlab/namespaces/#{group_id}/upgrade/#{plan_id}" + end + + def self.renew_subscription_url(group_id) + "#{self.subscriptions_url}/gitlab/namespaces/#{group_id}/renew" + end + def self.subscription_portal_admin_email ENV.fetch('SUBSCRIPTION_PORTAL_ADMIN_EMAIL', 'gl_com_api@gitlab.com') end diff --git a/lib/gitlab/template/gitlab_ci_yml_template.rb b/lib/gitlab/template/gitlab_ci_yml_template.rb index 263483ba54b..35f45c8809f 100644 --- a/lib/gitlab/template/gitlab_ci_yml_template.rb +++ b/lib/gitlab/template/gitlab_ci_yml_template.rb @@ -6,11 +6,7 @@ module Gitlab BASE_EXCLUDED_PATTERNS = [%r{\.latest\.}].freeze TEMPLATES_WITH_LATEST_VERSION = { - 'Jobs/Browser-Performance-Testing' => true, - 'Jobs/Build' => true, - 'Security/API-Fuzzing' => true, - 'Security/DAST' => true, - 'Terraform' => true + 'Jobs/Build' => true }.freeze def description diff --git a/lib/gitlab/throttle.rb b/lib/gitlab/throttle.rb index 622dc7d9ed0..384953533b5 100644 --- a/lib/gitlab/throttle.rb +++ b/lib/gitlab/throttle.rb @@ -7,7 +7,7 @@ module Gitlab # Each of these settings follows the same pattern of specifying separate # authenticated and unauthenticated rates via settings. New throttles should # ideally be regular as well. - REGULAR_THROTTLES = [:api, :packages_api, :files_api].freeze + REGULAR_THROTTLES = [:api, :packages_api, :files_api, :deprecated_api].freeze def self.settings Gitlab::CurrentSettings.current_application_settings diff --git a/lib/gitlab/tracking/docs/helper.rb b/lib/gitlab/tracking/docs/helper.rb deleted file mode 100644 index 4e03858b771..00000000000 --- a/lib/gitlab/tracking/docs/helper.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Tracking - module Docs - # Helper with functions to be used by HAML templates - module Helper - def auto_generated_comment - <<-MARKDOWN.strip_heredoc - --- - stage: Growth - group: Product Intelligence - info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#designated-technical-writers - --- - - <!--- - This documentation is auto generated by a script. - - Please do not edit this file directly, check generate_event_dictionary task on lib/tasks/gitlab/snowplow.rake. - ---> - - <!-- vale gitlab.Spelling = NO --> - MARKDOWN - end - - def render_description(object) - return 'Missing description' unless object.description.present? - - object.description - end - - def render_event_taxonomy(object) - headers = %w[category action label property value] - values = %i[category action label property_description value_description] - values = values.map { |key| backtick(object.attributes[key]) } - values = values.join(" | ") - - [ - "| #{headers.join(" | ")} |", - "#{'|---' * headers.size}|", - "| #{values} |" - ].join("\n") - end - - def md_link_to(anchor_text, url) - "[#{anchor_text}](#{url})" - end - - def render_owner(object) - "Owner: #{backtick(object.product_group)}" - end - - def render_tiers(object) - "Tiers: #{object.tiers.map(&method(:backtick)).join(', ')}" - end - - def render_yaml_definition_path(object) - "YAML definition: #{backtick(object.yaml_path)}" - end - - def backtick(string) - "`#{string}`" - end - end - end - end -end diff --git a/lib/gitlab/tracking/docs/renderer.rb b/lib/gitlab/tracking/docs/renderer.rb deleted file mode 100644 index 184b935c2ba..00000000000 --- a/lib/gitlab/tracking/docs/renderer.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Tracking - module Docs - class Renderer - include Gitlab::Tracking::Docs::Helper - DICTIONARY_PATH = Rails.root.join('doc', 'development', 'snowplow') - TEMPLATE_PATH = Rails.root.join('lib', 'gitlab', 'tracking', 'docs', 'templates', 'default.md.haml') - - def initialize(event_definitions) - @layout = Haml::Engine.new(File.read(TEMPLATE_PATH)) - @event_definitions = event_definitions.sort - end - - def contents - # Render and remove an extra trailing new line - @contents ||= @layout.render(self, event_definitions: @event_definitions).sub!(/\n(?=\Z)/, '') - end - - def write - filename = DICTIONARY_PATH.join('dictionary.md').to_s - - FileUtils.mkdir_p(DICTIONARY_PATH) - File.write(filename, contents) - - filename - end - end - end - end -end diff --git a/lib/gitlab/tracking/docs/templates/default.md.haml b/lib/gitlab/tracking/docs/templates/default.md.haml deleted file mode 100644 index 568f56590fa..00000000000 --- a/lib/gitlab/tracking/docs/templates/default.md.haml +++ /dev/null @@ -1,35 +0,0 @@ -= auto_generated_comment - -:plain - # Event Dictionary - - This file is autogenerated, please do not edit it directly. - - To generate these files from the GitLab repository, run: - - ```shell - bundle exec rake gitlab:snowplow:generate_event_dictionary - ``` - - The Event Dictionary is based on the following event definition YAML files: - - - [`config/events`](https://gitlab.com/gitlab-org/gitlab/-/tree/f9a404301ca22d038e7b9a9eb08d9c1bbd6c4d84/config/events) - - [`ee/config/events`](https://gitlab.com/gitlab-org/gitlab/-/tree/f9a404301ca22d038e7b9a9eb08d9c1bbd6c4d84/ee/config/events) - - ## Event definitions - -\ -- event_definitions.each do |_path, object| - - = "### `#{object.category} #{object.action}`" - \ - = render_event_taxonomy(object) - \ - = render_description(object) - \ - = render_yaml_definition_path(object) - \ - = render_owner(object) - \ - = render_tiers(object) - \ diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index fe5669be014..df62e8bbbe6 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -3,13 +3,14 @@ module Gitlab module Tracking class StandardContext - GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-5' + GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-7' GITLAB_RAILS_SOURCE = 'gitlab-rails' def initialize(namespace: nil, project: nil, user: nil, **extra) @namespace = namespace @plan = namespace&.actual_plan_name @project = project + @user = user @extra = extra end @@ -35,7 +36,7 @@ module Gitlab private - attr_accessor :namespace, :project, :extra, :plan + attr_accessor :namespace, :project, :extra, :plan, :user def to_h { @@ -44,6 +45,7 @@ module Gitlab plan: plan, extra: extra }.merge(project_and_namespace) + .merge(user_data) end def project_and_namespace @@ -58,6 +60,10 @@ module Gitlab def project_id project.is_a?(Integer) ? project : project&.id end + + def user_data + ::Feature.enabled?(:add_actor_based_user_to_snowplow_tracking, user) ? { user_id: user&.id } : {} + end end end end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index db0cb4c6326..6e5196ecdbd 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -6,6 +6,7 @@ module Gitlab METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') BASE_REPO_PATH = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze + AVAILABLE_STATUSES = %w[active data_available implemented deprecated].freeze InvalidError = Class.new(RuntimeError) @@ -59,6 +60,10 @@ module Gitlab attributes[:data_category]&.downcase! end + def available? + AVAILABLE_STATUSES.include?(attributes[:status]) + end + alias_method :to_dictionary, :to_h class << self @@ -76,7 +81,7 @@ module Gitlab end def with_instrumentation_class - all.select { |definition| definition.attributes[:instrumentation_class].present? } + all.select { |definition| definition.attributes[:instrumentation_class].present? && definition.available? } end def schemer diff --git a/lib/gitlab/usage/metrics/instrumentations/active_user_count_metric.rb b/lib/gitlab/usage/metrics/instrumentations/active_user_count_metric.rb new file mode 100644 index 00000000000..2f3b3af306f --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/active_user_count_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class ActiveUserCountMetric < DatabaseMetric + operation :count + + relation { User.active } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_users_associating_milestones_to_releases_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_users_associating_milestones_to_releases_metric.rb new file mode 100644 index 00000000000..c10182e23aa --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_users_associating_milestones_to_releases_metric.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountUsersAssociatingMilestonesToReleasesMetric < DatabaseMetric + operation :distinct_count, column: :author_id + + relation { Release.with_milestones } + + start { Release.minimum(:author_id) } + finish { Release.maximum(:author_id) } + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 854242031be..dd66f9133bb 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -76,7 +76,7 @@ module Gitlab hostname: add_metric('HostnameMetric'), version: alt_usage_data { Gitlab::VERSION }, installation_type: alt_usage_data { installation_type }, - active_user_count: count(User.active), + active_user_count: add_metric('ActiveUserCountMetric'), edition: 'CE' } end @@ -123,17 +123,9 @@ module Gitlab clusters_platforms_eks: count(::Clusters::Cluster.aws_installed.enabled), clusters_platforms_gke: count(::Clusters::Cluster.gcp_installed.enabled), clusters_platforms_user: count(::Clusters::Cluster.user_provided.enabled), - clusters_applications_helm: count(::Clusters::Applications::Helm.available), - clusters_applications_ingress: count(::Clusters::Applications::Ingress.available), - clusters_applications_cert_managers: count(::Clusters::Applications::CertManager.available), - clusters_applications_crossplane: count(::Clusters::Applications::Crossplane.available), - clusters_applications_prometheus: count(::Clusters::Applications::Prometheus.available), - clusters_applications_runner: count(::Clusters::Applications::Runner.available), - clusters_applications_knative: count(::Clusters::Applications::Knative.available), - clusters_applications_elastic_stack: count(::Clusters::Applications::ElasticStack.available), - clusters_applications_jupyter: count(::Clusters::Applications::Jupyter.available), - clusters_applications_cilium: count(::Clusters::Applications::Cilium.available), clusters_management_project: count(::Clusters::Cluster.with_management_project), + clusters_integrations_elastic_stack: count(::Clusters::Integrations::ElasticStack.enabled), + clusters_integrations_prometheus: count(::Clusters::Integrations::Prometheus.enabled), kubernetes_agents: count(::Clusters::Agent), kubernetes_agents_with_token: distinct_count(::Clusters::AgentToken, :agent_id), in_review_folder: count(::Environment.in_review_folder), @@ -211,19 +203,6 @@ module Gitlab } end - def snowplow_event_counts(time_period) - return {} unless report_snowplow_events? - - { - promoted_issues: count( - self_monitoring_project - .product_analytics_events - .by_category_and_action('epics', 'promote') - .where(time_period) - ) - } - end - def system_usage_data_monthly { counts_monthly: { @@ -236,10 +215,9 @@ module Gitlab packages: count(::Packages::Package.where(monthly_time_range_db_params)), personal_snippets: count(PersonalSnippet.where(monthly_time_range_db_params)), project_snippets: count(ProjectSnippet.where(monthly_time_range_db_params)), - projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(monthly_time_range_db_params), :project_id) - }.merge( - snowplow_event_counts(monthly_time_range_db_params(column: :collector_tstamp)) - ).tap do |data| + projects_with_alerts_created: distinct_count(::AlertManagement::Alert.where(monthly_time_range_db_params), :project_id), + promoted_issues: DEPRECATED_VALUE + }.tap do |data| data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end } @@ -412,7 +390,6 @@ module Gitlab response[:"projects_#{name}_active"] = count(Integration.active.where.not(project: nil).where(type: type)) response[:"groups_#{name}_active"] = count(Integration.active.where.not(group: nil).where(type: type)) - response[:"templates_#{name}_active"] = count(Integration.active.where(template: true, type: type)) response[:"instances_#{name}_active"] = count(Integration.active.where(instance: true, type: type)) response[:"projects_inheriting_#{name}_active"] = count(Integration.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: type)) response[:"groups_inheriting_#{name}_active"] = count(Integration.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: type)) @@ -523,10 +500,6 @@ module Gitlab # rubocop: disable UsageData/LargeTable def usage_activity_by_stage_configure(time_period) { - clusters_applications_cert_managers: cluster_applications_user_distinct_count(::Clusters::Applications::CertManager, time_period), - clusters_applications_helm: cluster_applications_user_distinct_count(::Clusters::Applications::Helm, time_period), - clusters_applications_ingress: cluster_applications_user_distinct_count(::Clusters::Applications::Ingress, time_period), - clusters_applications_knative: cluster_applications_user_distinct_count(::Clusters::Applications::Knative, time_period), clusters_management_project: clusters_user_distinct_count(::Clusters::Cluster.with_management_project, time_period), clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled, time_period), clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled, time_period), @@ -621,7 +594,7 @@ module Gitlab { clusters: distinct_count(::Clusters::Cluster.where(time_period), :user_id), - clusters_applications_prometheus: cluster_applications_user_distinct_count(::Clusters::Applications::Prometheus, time_period), + clusters_integrations_prometheus: cluster_integrations_user_distinct_count(::Clusters::Integrations::Prometheus, time_period), operations_dashboard_default_dashboard: count(::User.active.with_dashboard('operations').where(time_period), start: minimum_id(User), finish: maximum_id(User)), @@ -647,7 +620,7 @@ module Gitlab # Omitted because of encrypted properties: `projects_jira_cloud_active`, `projects_jira_server_active` # rubocop: disable CodeReuse/ActiveRecord def usage_activity_by_stage_plan(time_period) - time_frame = time_period.present? ? '28d' : 'none' + time_frame = metric_time_period(time_period) { issues: add_metric('CountUsersCreatingIssuesMetric', time_frame: time_frame), notes: distinct_count(::Note.where(time_period), :author_id), @@ -665,11 +638,13 @@ module Gitlab # Omitted because no user, creator or author associated: `environments`, `feature_flags`, `in_review_folder`, `pages_domains` # rubocop: disable CodeReuse/ActiveRecord def usage_activity_by_stage_release(time_period) + time_frame = metric_time_period(time_period) { deployments: distinct_count(::Deployment.where(time_period), :user_id), failed_deployments: distinct_count(::Deployment.failed.where(time_period), :user_id), releases: distinct_count(::Release.where(time_period), :author_id), - successful_deployments: distinct_count(::Deployment.success.where(time_period), :user_id) + successful_deployments: distinct_count(::Deployment.success.where(time_period), :user_id), + releases_with_milestones: add_metric('CountUsersAssociatingMilestonesToReleasesMetric', time_frame: time_frame) } end # rubocop: enable CodeReuse/ActiveRecord @@ -685,8 +660,7 @@ module Gitlab ci_pipeline_config_repository: distinct_count(::Ci::Pipeline.repository_source.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)), ci_pipeline_schedules: distinct_count(::Ci::PipelineSchedule.where(time_period), :owner_id), ci_pipelines: distinct_count(::Ci::Pipeline.where(time_period), :user_id, start: minimum_id(User), finish: maximum_id(User)), - ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id), - clusters_applications_runner: cluster_applications_user_distinct_count(::Clusters::Applications::Runner, time_period) + ci_triggers: distinct_count(::Ci::Trigger.where(time_period), :owner_id) } end # rubocop: enable CodeReuse/ActiveRecord @@ -755,6 +729,10 @@ module Gitlab private + def metric_time_period(time_period) + time_period.present? ? '28d' : 'none' + end + def gitaly_apdex with_prometheus_client(verify: false, fallback: FALLBACK) do |client| result = client.query('avg_over_time(gitlab_usage_ping:gitaly_apdex:ratio_avg_over_time_5m[1w])').first @@ -794,10 +772,6 @@ module Gitlab } end - def report_snowplow_events? - self_monitoring_project && Feature.enabled?(:product_analytics_tracking, type: :ops) - end - def distinct_count_service_desk_enabled_projects(time_period) project_creator_id_start = minimum_id(User) project_creator_id_finish = maximum_id(User) @@ -858,17 +832,13 @@ module Gitlab count(::Issue.with_prometheus_alert_events, start: minimum_id(Issue), finish: maximum_id(Issue)) end - def self_monitoring_project - Gitlab::CurrentSettings.self_monitoring_project - end - def clear_memoized CE_MEMOIZED_VALUES.each { |v| clear_memoization(v) } end # rubocop: disable CodeReuse/ActiveRecord - def cluster_applications_user_distinct_count(applications, time_period) - distinct_count(applications.where(time_period).available.joins(:cluster), 'clusters.user_id') + def cluster_integrations_user_distinct_count(integrations, time_period) + distinct_count(integrations.where(time_period).enabled.joins(:cluster), 'clusters.user_id') end def clusters_user_distinct_count(clusters, time_period) diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index e5a50c92329..b8de7de848d 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -5,23 +5,14 @@ module Gitlab::UsageDataCounters REDIS_SLOT = 'ci_templates' KNOWN_EVENTS_FILE_PATH = File.expand_path('known_events/ci_templates.yml', __dir__) - # NOTE: Events originating from implicit Auto DevOps pipelines get prefixed with `implicit_` - TEMPLATE_TO_EVENT = { - '5-Minute-Production-App.gitlab-ci.yml' => '5_min_production_app', - 'Auto-DevOps.gitlab-ci.yml' => 'auto_devops', - 'AWS/CF-Provision-and-Deploy-EC2.gitlab-ci.yml' => 'aws_cf_deploy_ec2', - 'AWS/Deploy-ECS.gitlab-ci.yml' => 'aws_deploy_ecs', - 'Jobs/Build.gitlab-ci.yml' => 'auto_devops_build', - 'Jobs/Deploy.gitlab-ci.yml' => 'auto_devops_deploy', - 'Jobs/Deploy.latest.gitlab-ci.yml' => 'auto_devops_deploy_latest', - 'Security/SAST.gitlab-ci.yml' => 'security_sast', - 'Security/Secret-Detection.gitlab-ci.yml' => 'security_secret_detection', - 'Terraform/Base.latest.gitlab-ci.yml' => 'terraform_base_latest' - }.freeze - class << self def track_unique_project_event(project_id:, template:, config_source:) - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(ci_template_event_name(template, config_source), values: project_id) + expanded_template_name = expand_template_name(template) + return unless expanded_template_name + + Gitlab::UsageDataCounters::HLLRedisCounter.track_event( + ci_template_event_name(expanded_template_name, config_source), values: project_id + ) end def ci_templates(relative_base = 'lib/gitlab/ci/templates') @@ -30,9 +21,12 @@ module Gitlab::UsageDataCounters def ci_template_event_name(template_name, config_source) prefix = 'implicit_' if config_source.to_s == 'auto_devops_source' - template_event_name = TEMPLATE_TO_EVENT[template_name] || template_to_event_name(template_name) - "p_#{REDIS_SLOT}_#{prefix}#{template_event_name}" + "p_#{REDIS_SLOT}_#{prefix}#{template_to_event_name(template_name)}" + end + + def expand_template_name(template_name) + Gitlab::Template::GitlabCiYmlTemplate.find(template_name.chomp('.gitlab-ci.yml'))&.full_name end private diff --git a/lib/gitlab/usage_data_counters/guest_package_events.yml b/lib/gitlab/usage_data_counters/guest_package_events.yml deleted file mode 100644 index a9b9f8ea235..00000000000 --- a/lib/gitlab/usage_data_counters/guest_package_events.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -- i_package_composer_guest_delete -- i_package_composer_guest_pull -- i_package_composer_guest_push -- i_package_conan_guest_delete -- i_package_conan_guest_pull -- i_package_conan_guest_push -- i_package_container_guest_delete -- i_package_container_guest_pull -- i_package_container_guest_push -- i_package_debian_guest_delete -- i_package_debian_guest_pull -- i_package_debian_guest_push -- i_package_generic_guest_delete -- i_package_generic_guest_pull -- i_package_generic_guest_push -- i_package_golang_guest_delete -- i_package_golang_guest_pull -- i_package_golang_guest_push -- i_package_maven_guest_delete -- i_package_maven_guest_pull -- i_package_maven_guest_push -- i_package_npm_guest_delete -- i_package_npm_guest_pull -- i_package_npm_guest_push -- i_package_nuget_guest_delete -- i_package_nuget_guest_pull -- i_package_nuget_guest_push -- i_package_pypi_guest_delete -- i_package_pypi_guest_pull -- i_package_pypi_guest_push -- i_package_tag_guest_delete -- i_package_tag_guest_pull -- i_package_tag_guest_push diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index cf790767f17..99bdd3ca9e9 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -1,44 +1,8 @@ -# Implicit Auto DevOps pipeline events -- name: p_ci_templates_implicit_auto_devops - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - -# Explicit include:template pipeline events -- name: p_ci_templates_5_min_production_app - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - -- name: p_ci_templates_aws_cf_deploy_ec2 - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - -- name: p_ci_templates_auto_devops_build - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - -- name: p_ci_templates_auto_devops_deploy - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - -- name: p_ci_templates_auto_devops_deploy_latest - category: ci_templates - redis_slot: ci_templates - aggregation: weekly - -# This part of the file is generated automatically by +# This file is generated automatically by # bin/rake gitlab:usage_data:generate_ci_template_events # # Do not edit it manually! -# -# The section above this should be removed once we roll out tracking all ci -# templates -# https://gitlab.com/gitlab-org/gitlab/-/issues/339684 - +--- - name: p_ci_templates_terraform_base_latest category: ci_templates redis_slot: ci_templates @@ -463,6 +427,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_auto_devops + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_jobs_dast_default_branch_deploy category: ci_templates redis_slot: ci_templates @@ -499,11 +467,11 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_auto_devops_deploy +- name: p_ci_templates_implicit_jobs_deploy category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_auto_devops_build +- name: p_ci_templates_implicit_jobs_build category: ci_templates redis_slot: ci_templates aggregation: weekly @@ -515,7 +483,7 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly -- name: p_ci_templates_implicit_auto_devops_deploy_latest +- name: p_ci_templates_implicit_jobs_deploy_latest category: ci_templates redis_slot: ci_templates aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 261d3b37783..feebc7f395a 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -149,7 +149,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_i_testing_test_case_parsed - name: i_testing_metrics_report_widget_total category: testing redis_slot: testing @@ -158,7 +157,6 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_i_testing_group_code_coverage_visit_total - name: i_testing_full_code_quality_report_total category: testing redis_slot: testing @@ -179,12 +177,10 @@ category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_i_testing_metrics_report_artifact_uploaders - name: i_testing_summary_widget_total category: testing redis_slot: testing aggregation: weekly - feature_flag: usage_data_i_testing_summary_widget_total # Project Management group - name: g_project_management_issue_title_changed category: issues_edit diff --git a/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml index 281db441829..3879c561cc4 100644 --- a/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml @@ -7,16 +7,13 @@ category: epic_boards_usage redis_slot: project_management aggregation: daily - feature_flag: track_epic_boards_activity - name: g_project_management_users_viewing_epic_boards category: epic_boards_usage redis_slot: project_management aggregation: daily - feature_flag: track_epic_boards_activity - name: g_project_management_users_updating_epic_board_names category: epic_boards_usage redis_slot: project_management aggregation: daily - feature_flag: track_epic_boards_activity diff --git a/lib/gitlab/usage_data_counters/known_events/importer_events.yml b/lib/gitlab/usage_data_counters/known_events/importer_events.yml new file mode 100644 index 00000000000..79bbac229bc --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/importer_events.yml @@ -0,0 +1,17 @@ +--- +# Importer events +- name: github_import_project_start + category: importer + redis_slot: import + aggregation: weekly + feature_flag: track_importer_activity +- name: github_import_project_success + category: importer + redis_slot: import + aggregation: weekly + feature_flag: track_importer_activity +- name: github_import_project_failure + category: importer + redis_slot: import + aggregation: weekly + feature_flag: track_importer_activity diff --git a/lib/gitlab/utils/delegator_override.rb b/lib/gitlab/utils/delegator_override.rb new file mode 100644 index 00000000000..15ba29d3916 --- /dev/null +++ b/lib/gitlab/utils/delegator_override.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + # This module is to validate that delegator classes (`SimpleDelegator`) do not + # accidentally override important logic on the fabricated object. + module DelegatorOverride + def delegator_target(target_class) + return unless ENV['STATIC_VERIFICATION'] + + unless self < ::SimpleDelegator + raise ArgumentError, "'#{self}' is not a subclass of 'SimpleDelegator' class." + end + + DelegatorOverride.validator(self).add_target(target_class) + end + + def delegator_override(*names) + return unless ENV['STATIC_VERIFICATION'] + raise TypeError unless names.all? { |n| n.is_a?(Symbol) } + + DelegatorOverride.validator(self).add_allowlist(names) + end + + def delegator_override_with(mod) + return unless ENV['STATIC_VERIFICATION'] + raise TypeError unless mod.is_a?(Module) + + DelegatorOverride.validator(self).add_allowlist(mod.instance_methods) + end + + def self.validator(delegator_class) + validators[delegator_class] ||= Validator.new(delegator_class) + end + + def self.validators + @validators ||= {} + end + + def self.verify! + validators.each_value do |validator| + validator.expand_on_ancestors(validators) + validator.validate_overrides! + end + end + end + end +end diff --git a/lib/gitlab/utils/delegator_override/error.rb b/lib/gitlab/utils/delegator_override/error.rb new file mode 100644 index 00000000000..dfe8d5468b4 --- /dev/null +++ b/lib/gitlab/utils/delegator_override/error.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + module DelegatorOverride + class Error + attr_accessor :method_name, :target_class, :target_location, :delegator_class, :delegator_location + + def initialize(method_name, target_class, target_location, delegator_class, delegator_location) + @method_name = method_name + @target_class = target_class + @target_location = target_location + @delegator_class = delegator_class + @delegator_location = delegator_location + end + + def to_s + "#{delegator_class}##{method_name} is overriding #{target_class}##{method_name}. delegator_location: #{delegator_location} target_location: #{target_location}" + end + end + end + end +end diff --git a/lib/gitlab/utils/delegator_override/validator.rb b/lib/gitlab/utils/delegator_override/validator.rb new file mode 100644 index 00000000000..402154b41c2 --- /dev/null +++ b/lib/gitlab/utils/delegator_override/validator.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + module DelegatorOverride + class Validator + UnexpectedDelegatorOverrideError = Class.new(StandardError) + + attr_reader :delegator_class, :target_classes + + OVERRIDE_ERROR_MESSAGE = <<~EOS + We've detected that the delegator is overriding a specific method(s) on the target class. + Please make sure if it's intentional and handle this error accordingly. + See https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/presenters/README.md#validate-accidental-overrides for more information. + EOS + + def initialize(delegator_class) + @delegator_class = delegator_class + @target_classes = [] + end + + def add_allowlist(names) + allowed_method_names.concat(names) + end + + def allowed_method_names + @allowed_method_names ||= [] + end + + def add_target(target_class) + @target_classes << target_class if target_class + end + + # This will make sure allowlist we put into ancestors are all included + def expand_on_ancestors(validators) + delegator_class.ancestors.each do |ancestor| + next if delegator_class == ancestor # ancestor includes itself + + validator_ancestor = validators[ancestor] + + next unless validator_ancestor + + add_allowlist(validator_ancestor.allowed_method_names) + end + end + + def validate_overrides! + return if target_classes.empty? + + errors = [] + + # Workaround to fully load the instance methods in the target class. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/69823#note_678887402 + begin + target_classes.map(&:new) + rescue ArgumentError + # Some models might raise ArgumentError here, but it's fine in this case, + # because this is enough to force ActiveRecord to generate the methods we + # need to verify, so it's safe to ignore it. + end + + (delegator_class.instance_methods - allowlist).each do |method_name| + target_classes.each do |target_class| + next unless target_class.instance_methods.include?(method_name) + + errors << generate_error(method_name, target_class, delegator_class) + end + end + + return if errors.empty? + + details = errors.map { |error| "- #{error}" }.join("\n") + + raise UnexpectedDelegatorOverrideError, + <<~TEXT + #{OVERRIDE_ERROR_MESSAGE} + Here are the conflict details. + + #{details} + TEXT + end + + private + + def generate_error(method_name, target_class, delegator_class) + target_location = extract_location(target_class, method_name) + delegator_location = extract_location(delegator_class, method_name) + Error.new(method_name, target_class, target_location, delegator_class, delegator_location) + end + + def extract_location(klass, method_name) + klass.instance_method(method_name).source_location&.join(':') || 'unknown' + end + + def allowlist + [].tap do |allowed| + allowed.concat(allowed_method_names) + allowed.concat(Object.instance_methods) + allowed.concat(::Delegator.instance_methods) + end + end + end + end + end +end diff --git a/lib/gitlab/verify/uploads.rb b/lib/gitlab/verify/uploads.rb index afcdbd087d2..0faf794e14d 100644 --- a/lib/gitlab/verify/uploads.rb +++ b/lib/gitlab/verify/uploads.rb @@ -28,7 +28,7 @@ module Gitlab end def actual_checksum(upload) - Upload.hexdigest(upload.absolute_path) + Upload.sha256_hexdigest(upload.absolute_path) end def remote_object_exists?(upload) diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index 9dc687f7740..3bacad72050 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -47,8 +47,18 @@ module Gitlab true end - def presents(name) - define_method(name) { subject } + def presents(*target_classes, as: nil) + if target_classes.any? { |k| k.is_a?(Symbol) } + raise ArgumentError, "Unsupported target class type: #{target_classes}." + end + + if self < ::Gitlab::View::Presenter::Delegated + target_classes.each { |k| delegator_target(k) } + elsif self < ::Gitlab::View::Presenter::Simple + # no-op + end + + define_method(as) { subject } if as end end end diff --git a/lib/gitlab/view/presenter/delegated.rb b/lib/gitlab/view/presenter/delegated.rb index d14f8cc4e5e..259cf0cf457 100644 --- a/lib/gitlab/view/presenter/delegated.rb +++ b/lib/gitlab/view/presenter/delegated.rb @@ -4,7 +4,18 @@ module Gitlab module View module Presenter class Delegated < SimpleDelegator + extend ::Gitlab::Utils::DelegatorOverride + + # TODO: Stop including auxiliary methods/modules in `Presenter::Base` as + # it overrides many methods in the Active Record models. + # See https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/presenters/README.md#validate-accidental-overrides + # for more information. include Gitlab::View::Presenter::Base + delegator_override_with Gitlab::Routing.url_helpers + delegator_override :can? + delegator_override :declarative_policy_delegate + delegator_override :present + delegator_override :web_url def initialize(subject, **attributes) @subject = subject diff --git a/lib/gitlab/with_feature_category.rb b/lib/gitlab/with_feature_category.rb deleted file mode 100644 index 65d21daf78a..00000000000 --- a/lib/gitlab/with_feature_category.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module WithFeatureCategory - extend ActiveSupport::Concern - include Gitlab::ClassAttributes - - class_methods do - def feature_category(category, actions = []) - feature_category_configuration[category] ||= [] - feature_category_configuration[category] += actions.map(&:to_s) - - validate_config!(feature_category_configuration) - end - - def feature_category_for_action(action) - category_config = feature_category_configuration.find do |_, actions| - actions.empty? || actions.include?(action) - end - - category_config&.first || superclass_feature_category_for_action(action) - end - - private - - def validate_config!(config) - empty = config.find { |_, actions| actions.empty? } - duplicate_actions = config.values.map(&:uniq).flatten.group_by(&:itself).select { |_, v| v.count > 1 }.keys - - if config.length > 1 && empty - raise ArgumentError, "#{empty.first} is defined for all actions, but other categories are set" - end - - if duplicate_actions.any? - raise ArgumentError, "Actions have multiple feature categories: #{duplicate_actions.join(', ')}" - end - end - - def feature_category_configuration - class_attributes[:feature_category_config] ||= {} - end - - def superclass_feature_category_for_action(action) - return unless superclass.respond_to?(:feature_category_for_action) - - superclass.feature_category_for_action(action) - end - end - end -end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 0f33c3aa68e..c40aa2273aa 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -32,7 +32,8 @@ module Gitlab GitalyServer: { address: Gitlab::GitalyClient.address(repository.storage), token: Gitlab::GitalyClient.token(repository.storage), - features: Feature::Gitaly.server_feature_flags(repository.project) + features: Feature::Gitaly.server_feature_flags(repository.project), + sidechannel: Feature.enabled?(:workhorse_use_sidechannel, repository.project, default_enabled: :yaml) } } @@ -169,6 +170,18 @@ module Gitlab ] end + def send_dependency(token, url) + params = { + 'Header' => { Authorization: ["Bearer #{token}"] }, + 'Url' => url + } + + [ + SEND_DATA_HEADER, + "send-dependency:#{encode(params)}" + ] + end + def channel_websocket(channel) details = { 'Channel' => { diff --git a/lib/gitlab/x509/certificate.rb b/lib/gitlab/x509/certificate.rb new file mode 100644 index 00000000000..c7289a51b49 --- /dev/null +++ b/lib/gitlab/x509/certificate.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module X509 + class Certificate + CERT_REGEX = /-----BEGIN CERTIFICATE-----(?:.|\n)+?-----END CERTIFICATE-----/.freeze + + attr_reader :key, :cert, :ca_certs + + def key_string + key.to_s + end + + def cert_string + cert.to_pem + end + + def ca_certs_string + ca_certs.map(&:to_pem).join('\n') unless ca_certs.blank? + end + + def self.from_strings(key_string, cert_string, ca_certs_string = nil) + key = OpenSSL::PKey::RSA.new(key_string) + cert = OpenSSL::X509::Certificate.new(cert_string) + ca_certs = load_ca_certs_bundle(ca_certs_string) + + new(key, cert, ca_certs) + end + + def self.from_files(key_path, cert_path, ca_certs_path = nil) + ca_certs_string = File.read(ca_certs_path) if ca_certs_path + + from_strings(File.read(key_path), File.read(cert_path), ca_certs_string) + end + + # Returns an array of OpenSSL::X509::Certificate objects, empty array if none found + # + # Ruby OpenSSL::X509::Certificate.new will only load the first + # certificate if a bundle is presented, this allows to parse multiple certs + # in the same file + def self.load_ca_certs_bundle(ca_certs_string) + return [] unless ca_certs_string + + ca_certs_string.scan(CERT_REGEX).map do |ca_cert_string| + OpenSSL::X509::Certificate.new(ca_cert_string) + end + end + + def initialize(key, cert, ca_certs = nil) + @key = key + @cert = cert + @ca_certs = ca_certs + end + end + end +end |