diff options
Diffstat (limited to 'tooling')
-rw-r--r-- | tooling/config/CODEOWNERS.yml | 5 | ||||
-rw-r--r-- | tooling/danger/product_intelligence.rb | 67 | ||||
-rw-r--r-- | tooling/danger/project_helper.rb | 4 | ||||
-rw-r--r-- | tooling/danger/specs.rb | 37 | ||||
-rw-r--r-- | tooling/danger/stable_branch.rb | 138 | ||||
-rw-r--r-- | tooling/danger/user_types.rb | 31 | ||||
-rw-r--r-- | tooling/docs/deprecation_handling.rb | 2 | ||||
-rw-r--r-- | tooling/lib/tooling/helm3_client.rb | 5 | ||||
-rw-r--r-- | tooling/quality/test_level.rb | 1 |
9 files changed, 272 insertions, 18 deletions
diff --git a/tooling/config/CODEOWNERS.yml b/tooling/config/CODEOWNERS.yml index ed712f18956..07fddde056c 100644 --- a/tooling/config/CODEOWNERS.yml +++ b/tooling/config/CODEOWNERS.yml @@ -56,6 +56,11 @@ - 'token_access/' - 'pipelines/' - 'ci/runner/' + - 'config/events/' + - 'config/audit_events/' + - 'runner_token_expiration/' + - '*metadata_id_tokens*' + - '/app/assets/javascripts/invite_members/' patterns: - '%{keyword}' diff --git a/tooling/danger/product_intelligence.rb b/tooling/danger/product_intelligence.rb index 621a7b509b0..58e327408a1 100644 --- a/tooling/danger/product_intelligence.rb +++ b/tooling/danger/product_intelligence.rb @@ -4,15 +4,21 @@ module Tooling module Danger module ProductIntelligence + METRIC_DIRS = %w[lib/gitlab/usage/metrics/instrumentations ee/lib/gitlab/usage/metrics/instrumentations].freeze APPROVED_LABEL = 'product intelligence::approved' REVIEW_LABEL = 'product intelligence::review pending' CHANGED_FILES_MESSAGE = <<~MSG - For the following files, a review from the [Data team and Product Intelligence team](https://gitlab.com/groups/gitlab-org/analytics-section/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) is recommended - Please check the ~"product intelligence" [Service Ping guide](https://docs.gitlab.com/ee/development/service_ping/) or the [Snowplow guide](https://docs.gitlab.com/ee/development/snowplow/). + For the following files, a review from the [Data team and Product Intelligence team](https://gitlab.com/groups/gitlab-org/analytics-section/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) is recommended + Please check the ~"product intelligence" [Service Ping guide](https://docs.gitlab.com/ee/development/service_ping/) or the [Snowplow guide](https://docs.gitlab.com/ee/development/snowplow/). - For MR review guidelines, see the [Service Ping review guidelines](https://docs.gitlab.com/ee/development/service_ping/review_guidelines.html) or the [Snowplow review guidelines](https://docs.gitlab.com/ee/development/snowplow/review_guidelines.html). + For MR review guidelines, see the [Service Ping review guidelines](https://docs.gitlab.com/ee/development/service_ping/review_guidelines.html) or the [Snowplow review guidelines](https://docs.gitlab.com/ee/development/snowplow/review_guidelines.html). - %<changed_files>s + %<changed_files>s + + MSG + + CHANGED_SCOPE_MESSAGE = <<~MSG + The following metrics could be affected by the modified scopes and require ~"product intelligence" review: MSG @@ -33,8 +39,61 @@ module Tooling helper.labels_to_add.concat(labels_to_add) unless labels_to_add.empty? end + def check_affected_scopes! + metric_scope_list = metric_scope_affected + return if metric_scope_list.empty? + + warn CHANGED_SCOPE_MESSAGE + convert_to_table(metric_scope_list) + helper.labels_to_add.concat(missing_labels) unless missing_labels.empty? + end + private + def convert_to_table(items) + message = "Scope | Affected files |\n" + message += "--- | ----- |\n" + items.each_key do |scope| + affected_files = items[scope] + message += "`#{scope}`| `#{affected_files[0]}` |\n" + affected_files[1..]&.each do |file_name| + message += " | `#{file_name}` |\n" + end + end + message + end + + def metric_scope_affected + select_models(helper.modified_files).each_with_object(Hash.new { |h, k| h[k] = [] }) do |file_name, matched_files| + helper.changed_lines(file_name).each do |mod_line, _i| + next unless mod_line =~ /^\+\s+scope :\w+/ + + affected_scope = mod_line.match(/:\w+/) + next if affected_scope.nil? + + affected_class = File.basename(file_name, '.rb').split('_').map(&:capitalize).join + scope_name = "#{affected_class}.#{affected_scope[0][1..]}" + + each_metric do |metric_def| + next unless File.read(metric_def).include?("relation { #{scope_name}") + + matched_files[scope_name].push(metric_def) + end + end + end + end + + def select_models(files) + files.select do |f| + f.start_with?('app/models/', 'ee/app/models/') + end + end + + def each_metric(&block) + METRIC_DIRS.each do |dir| + Dir.glob(File.join(dir, '*.rb')).each(&block) + end + end + def missing_labels return [] unless helper.ci? diff --git a/tooling/danger/project_helper.rb b/tooling/danger/project_helper.rb index a69d9049035..fbf102422aa 100644 --- a/tooling/danger/project_helper.rb +++ b/tooling/danger/project_helper.rb @@ -98,14 +98,14 @@ module Tooling %r{\A((ee|jh)/)?db/(geo/)?(migrate|post_migrate)/} => [:database, :migration], %r{\A((ee|jh)/)?db/(?!fixtures)[^/]+} => [:database], - %r{\A((ee|jh)/)?lib/gitlab/(database|background_migration|sql|github_import)(/|\.rb)} => [:database, :backend], + %r{\A((ee|jh)/)?lib/gitlab/(database|background_migration|sql)(/|\.rb)} => [:database, :backend], %r{\A(app/services/authorized_project_update/find_records_due_for_refresh_service)(/|\.rb)} => [:database, :backend], %r{\A(app/models/project_authorization|app/services/users/refresh_authorized_projects_service)(/|\.rb)} => [:database, :backend], %r{\A((ee|jh)/)?app/finders/} => [:database, :backend], %r{\Arubocop/cop/migration(/|\.rb)} => :database, %r{\A(\.ruby-version\z|\.nvmrc\z|\.tool-versions\z)} => :tooling, - %r{\A(\.gitlab-ci\.yml\z|\.gitlab\/ci)} => :tooling, + %r{\A(\.gitlab-ci\.yml\z|\.gitlab/ci)} => :tooling, %r{\A\.codeclimate\.yml\z} => :tooling, %r{\Alefthook.yml\z} => :tooling, %r{\A\.editorconfig\z} => :tooling, diff --git a/tooling/danger/specs.rb b/tooling/danger/specs.rb index 6832f7d10d1..c7baf920314 100644 --- a/tooling/danger/specs.rb +++ b/tooling/danger/specs.rb @@ -45,6 +45,13 @@ module Tooling for background information and alternative options. SUGGEST_COMMENT + FEATURE_CATEGORY_REGEX = /^\+.?RSpec\.describe(.+)(?!feature_category)/.freeze + FEATURE_CATEGORY_SUGGESTION = <<~SUGGESTION_MARKDOWN + Consider adding `feature_category: <feature_category_name>` for this example if it is not set already. + See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#feature-category-metadata). + SUGGESTION_MARKDOWN + FEATURE_CATEGORY_EXCLUDE = 'feature_category' + def changed_specs_files(ee: :include) changed_files = helper.all_changed_files folder_prefix = @@ -64,8 +71,8 @@ module Tooling add_suggestion( filename, MATCH_WITH_ARRAY_REGEX, - MATCH_WITH_ARRAY_REPLACEMENT, - MATCH_WITH_ARRAY_SUGGESTION + MATCH_WITH_ARRAY_SUGGESTION, + MATCH_WITH_ARRAY_REPLACEMENT ) end @@ -73,19 +80,30 @@ module Tooling add_suggestion( filename, PROJECT_FACTORY_REGEX, - PROJECT_FACTORY_REPLACEMENT, - PROJECT_FACTORY_SUGGESTION + PROJECT_FACTORY_SUGGESTION, + PROJECT_FACTORY_REPLACEMENT + ) + end + + def add_suggestions_for_feature_category(filename) + add_suggestion( + filename, + FEATURE_CATEGORY_REGEX, + FEATURE_CATEGORY_SUGGESTION, + nil, + FEATURE_CATEGORY_EXCLUDE ) end private def added_lines_matching(filename, regex) - helper.changed_lines(filename).grep(/\A\+ /).grep(regex) + helper.changed_lines(filename).grep(/\A\+( )?/).grep(regex) end - def add_suggestion(filename, regex, replacement, comment_text) + def add_suggestion(filename, regex, comment_text, replacement = nil, exclude = nil) added_lines = added_lines_matching(filename, regex) + return if added_lines.empty? spec_file_lines = project_helper.file_lines(filename) @@ -93,9 +111,14 @@ module Tooling added_lines.each_with_object([]) do |added_line, processed_line_numbers| line_number = find_line_number(spec_file_lines, added_line.delete_prefix('+'), exclude_indexes: processed_line_numbers) next unless line_number + next if !exclude.nil? && added_line.include?(exclude) processed_line_numbers << line_number - text = format(comment(comment_text), suggested_line: spec_file_lines[line_number].gsub(regex, replacement)) + + suggested_line = spec_file_lines[line_number] + suggested_line = suggested_line.gsub(regex, replacement) unless replacement.nil? + + text = format(comment(comment_text), suggested_line: suggested_line) markdown(text, file: filename, line: line_number.succ) end end diff --git a/tooling/danger/stable_branch.rb b/tooling/danger/stable_branch.rb new file mode 100644 index 00000000000..6c0b94b4f06 --- /dev/null +++ b/tooling/danger/stable_branch.rb @@ -0,0 +1,138 @@ +# frozen_string_literal: true + +module Tooling + module Danger + module StableBranch + VersionApiError = Class.new(StandardError) + + STABLE_BRANCH_REGEX = %r{\A(?<version>\d+-\d+)-stable-ee\z}.freeze + + # rubocop:disable Lint/MixedRegexpCaptureTypes + VERSION_REGEX = %r{ + \A(?<major>\d+) + \.(?<minor>\d+) + (\.(?<patch>\d+))? + (-(?<rc>rc(?<rc_number>\d*)))? + (-\h+\.\h+)? + (-ee|\.ee\.\d+)?\z + }x.freeze + # rubocop:enable Lint/MixedRegexpCaptureTypes + + MAINTENANCE_POLICY_URL = 'https://docs.gitlab.com/ee/policy/maintenance.html' + + MAINTENANCE_POLICY_MESSAGE = <<~MSG + See the [release and maintenance policy](#{MAINTENANCE_POLICY_URL}) for more information. + MSG + + FEATURE_ERROR_MESSAGE = <<~MSG + This MR includes the `type::feature` label. Features do not qualify for patch releases. #{MAINTENANCE_POLICY_MESSAGE} + MSG + + BUG_ERROR_MESSAGE = <<~MSG + This branch is meant for backporting bug fixes. If this MR qualifies please add the `type::bug` label. #{MAINTENANCE_POLICY_MESSAGE} + MSG + + VERSION_ERROR_MESSAGE = <<~MSG + Patches are only being accepted on the most recent 3 minor versions of GitLab. #{MAINTENANCE_POLICY_MESSAGE} + MSG + + FAILED_VERSION_REQUEST_MESSAGE = <<~MSG + There was a problem checking if this is a qualified version for backporting. Re-running this job may fix the problem. + MSG + + # rubocop:disable Style/SignalException + def check! + return unless stable_target_branch && !helper.security_mr? + + fail FEATURE_ERROR_MESSAGE if has_feature_label? + fail BUG_ERROR_MESSAGE unless has_bug_label? + fail VERSION_ERROR_MESSAGE unless targeting_patchable_version? + end + # rubocop:enable Style/SignalException + + private + + def stable_target_branch + helper.mr_target_branch.match(STABLE_BRANCH_REGEX) + end + + def has_feature_label? + helper.mr_has_labels?('type::feature') + end + + def has_bug_label? + helper.mr_has_labels?('type::bug') + end + + def targeting_patchable_version? + raise VersionApiError if last_three_minor_versions.empty? + + last_three_minor_versions.include?(targeted_version) + rescue VersionApiError + # don't fail the job since we do not know the recent versions + warn FAILED_VERSION_REQUEST_MESSAGE + true + end + + def last_three_minor_versions + return [] unless versions + + current_version = versions.first.match(VERSION_REGEX) + version_1 = previous_minor_version(current_version) + version_2 = previous_minor_version(version_1) + + [ + version_to_minor_string(current_version), + version_to_minor_string(version_1), + version_to_minor_string(version_2) + ] + end + + def targeted_version + stable_target_branch[1].tr('-', '.') + end + + def versions(page = 1) + version_api_endpoint = "https://version.gitlab.com/api/v1/versions?per_page=50&page=#{page}" + response = HTTParty.get(version_api_endpoint) # rubocop:disable Gitlab/HTTParty + + raise VersionApiError unless response.success? + + version_list = response.parsed_response.map { |v| v['version'] } # rubocop:disable Rails/Pluck + + version_list.sort_by { |v| Gem::Version.new(v) }.reverse + end + + def previous_minor_version(version) + previous_minor = version[:minor].to_i - 1 + + return "#{version[:major]}.#{previous_minor}".match(VERSION_REGEX) if previous_minor >= 0 + + fetch_last_minor_version_for_major(version[:major].to_i - 1) + end + + def fetch_last_minor_version_for_major(major) + page = 1 + last_minor_version = nil + + while last_minor_version.nil? + last_minor_version = versions(page).find do |version| + version.split('.').first.to_i == major + end + + break if page > 10 + + page += 1 + end + + raise VersionApiError if last_minor_version.nil? + + last_minor_version.match(VERSION_REGEX) + end + + def version_to_minor_string(version) + "#{version[:major]}.#{version[:minor]}" + end + end + end +end diff --git a/tooling/danger/user_types.rb b/tooling/danger/user_types.rb new file mode 100644 index 00000000000..8320c43ae93 --- /dev/null +++ b/tooling/danger/user_types.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Tooling + module Danger + module UserTypes + FILE_PATH = "app/models/concerns/has_user_type.rb" + BOT_USER_TYPES_CHANGE_INDICATOR_REGEX = %r{BOT_USER_TYPES}.freeze + BOT_USER_TYPES_CHANGED_WARNING = <<~MSG + You are changing BOT_USER_TYPES in `app/models/concerns/has_user_type.rb`. + If you are adding or removing new bots, remember to update the `active_billable_users` index with the new value. + If the bot is not billable, remember to make sure that it's not counted as a billable user. + MSG + + def bot_user_types_change_warning + return unless impacted? + + warn BOT_USER_TYPES_CHANGED_WARNING if bot_user_types_impacted? + end + + private + + def impacted? + helper.modified_files.include?(FILE_PATH) + end + + def bot_user_types_impacted? + helper.changed_lines(FILE_PATH).any? { |change| change =~ BOT_USER_TYPES_CHANGE_INDICATOR_REGEX } + end + end + end +end diff --git a/tooling/docs/deprecation_handling.rb b/tooling/docs/deprecation_handling.rb index a620eac4c91..bcdf73e0044 100644 --- a/tooling/docs/deprecation_handling.rb +++ b/tooling/docs/deprecation_handling.rb @@ -22,7 +22,7 @@ module Docs entries = source_file_paths.flat_map do |file| YAML.load_file(file) end - entries = entries.sort_by { |d| d["name"] } + entries = entries.sort_by { |d| d["title"] } milestones = entries.map { |entry| entry[milestone_key_name] }.uniq milestones = VersionSorter.rsort(milestones) diff --git a/tooling/lib/tooling/helm3_client.rb b/tooling/lib/tooling/helm3_client.rb index d83dbeac76b..d4e7faa802e 100644 --- a/tooling/lib/tooling/helm3_client.rb +++ b/tooling/lib/tooling/helm3_client.rb @@ -35,10 +35,7 @@ module Tooling end def delete(release_name:) - run_command([ - 'uninstall', - release_name - ]) + run_command(['uninstall', release_name]) end private diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb index d630ffd5432..29da7dddd03 100644 --- a/tooling/quality/test_level.rb +++ b/tooling/quality/test_level.rb @@ -19,6 +19,7 @@ module Quality bin channels config + contracts db dependencies elastic |