diff options
Diffstat (limited to 'tooling')
41 files changed, 1287 insertions, 452 deletions
diff --git a/tooling/bin/find_changes b/tooling/bin/find_changes deleted file mode 100755 index 38e1f363dd9..00000000000 --- a/tooling/bin/find_changes +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'gitlab' - -class FindChanges # rubocop:disable Gitlab/NamespacedClass - def initialize(output_file:, matched_tests_file: nil, frontend_fixtures_mapping_path: nil) - @gitlab_token = ENV.fetch('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE', '') - @gitlab_endpoint = ENV.fetch('CI_API_V4_URL') - @mr_project_path = ENV.fetch('CI_MERGE_REQUEST_PROJECT_PATH') - @mr_iid = ENV.fetch('CI_MERGE_REQUEST_IID') - @output_file = output_file - @matched_tests_file = matched_tests_file - @frontend_fixtures_mapping_path = frontend_fixtures_mapping_path - end - - def execute - add_frontend_fixture_files! - - File.write(output_file, file_changes.join(' ')) - end - - private - - def add_frontend_fixture_files? - matched_tests_file && frontend_fixtures_mapping_path - end - - def add_frontend_fixture_files! - return unless add_frontend_fixture_files? - - # If we have a `test file -> JSON frontend fixture` mapping file, we add the files JSON frontend fixtures - # files to the list of changed files so that Jest can automatically run the dependent tests thanks to --findRelatedTests - test_files.each do |test_file| - file_changes.concat(frontend_fixtures_mapping[test_file]) if frontend_fixtures_mapping.key?(test_file) - end - end - - def file_changes - @file_changes ||= - if File.exist?(output_file) - File.read(output_file).split(' ') - else - Gitlab.configure do |config| - config.endpoint = gitlab_endpoint - config.private_token = gitlab_token - end - - mr_changes.changes.flat_map do |change| - change.to_h.values_at('old_path', 'new_path') - end.uniq - end - end - - def mr_changes - @mr_changes ||= Gitlab.merge_request_changes(mr_project_path, mr_iid) - end - - def test_files - return [] if !matched_tests_file || !File.exist?(matched_tests_file) - - File.read(matched_tests_file).split(' ') - end - - def frontend_fixtures_mapping - return {} if !frontend_fixtures_mapping_path || !File.exist?(frontend_fixtures_mapping_path) - - JSON.parse(File.read(frontend_fixtures_mapping_path)) # rubocop:disable Gitlab/Json - end - - attr_reader :gitlab_token, :gitlab_endpoint, :mr_project_path, :mr_iid, :output_file, :matched_tests_file, :frontend_fixtures_mapping_path -end - -output_file = ARGV.shift -raise ArgumentError, "An path to an output file must be given as first argument of #{__FILE__}." if output_file.nil? - -matched_tests_file = ARGV.shift -frontend_fixtures_mapping_path = ARGV.shift - -FindChanges - .new(output_file: output_file, matched_tests_file: matched_tests_file, frontend_fixtures_mapping_path: frontend_fixtures_mapping_path) - .execute diff --git a/tooling/bin/find_only_allowed_files_changes b/tooling/bin/find_only_allowed_files_changes new file mode 100755 index 00000000000..c40048c66fa --- /dev/null +++ b/tooling/bin/find_only_allowed_files_changes @@ -0,0 +1,12 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative '../lib/tooling/find_changes' + +if Tooling::FindChanges.new(from: :api).only_allowed_files_changed + puts "Only files with extensions #{ALLOWED_FILE_TYPES.join(', ')} were changed" + exit 0 +else + puts "Changes were made to files with extensions other than #{ALLOWED_FILE_TYPES.join(', ')}" + exit 1 +end diff --git a/tooling/bin/find_tests b/tooling/bin/find_tests deleted file mode 100755 index 33834064f36..00000000000 --- a/tooling/bin/find_tests +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'test_file_finder' - -changes = ARGV.shift -output_file = ARGV.shift - -changed_files = File.read(changes).split(' ') - -tff = TestFileFinder::FileFinder.new(paths: changed_files).tap do |file_finder| - file_finder.use TestFileFinder::MappingStrategies::PatternMatching.load('tests.yml') - - if ENV['RSPEC_TESTS_MAPPING_ENABLED'] - file_finder.use TestFileFinder::MappingStrategies::DirectMatching.load_json(ENV['RSPEC_TESTS_MAPPING_PATH']) - end -end - -File.write(output_file, tff.test_files.uniq.join(' ')) diff --git a/tooling/bin/gettext_extractor b/tooling/bin/gettext_extractor new file mode 100755 index 00000000000..39f029616df --- /dev/null +++ b/tooling/bin/gettext_extractor @@ -0,0 +1,29 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative '../lib/tooling/gettext_extractor' + +pot_file = ARGV.shift + +if !pot_file || !Dir.exist?(File.dirname(pot_file)) + abort <<~MSG + Please provide a target file name as the first argument, e.g. + #{$PROGRAM_NAME} locale/gitlab.pot + MSG +end + +puts <<~MSG + Extracting translatable strings from source files... +MSG + +root_dir = File.expand_path('../../', __dir__) + +extractor = Tooling::GettextExtractor.new( + glob_base: root_dir +) + +File.write(pot_file, extractor.generate_pot) + +puts <<~MSG + All done. Please commit the changes to `#{pot_file}`. +MSG diff --git a/tooling/bin/js_to_system_specs_mappings b/tooling/bin/js_to_system_specs_mappings deleted file mode 100755 index 3e9d9cb4c5f..00000000000 --- a/tooling/bin/js_to_system_specs_mappings +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require_relative '../lib/tooling/mappings/js_to_system_specs_mappings' - -changes = ARGV.shift -matching_tests = ARGV.shift - -changed_files = File.read(changes).split(' ') -matching_test_files = File.read(matching_tests).split(' ') - -system_tests = Tooling::Mappings::JsToSystemSpecsMappings.new.execute(changed_files) - -File.write(matching_tests, (matching_test_files + system_tests).join(' ')) diff --git a/tooling/bin/predictive_tests b/tooling/bin/predictive_tests new file mode 100755 index 00000000000..61543494a69 --- /dev/null +++ b/tooling/bin/predictive_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative '../lib/tooling/predictive_tests' + +Tooling::PredictiveTests.new.execute diff --git a/tooling/bin/view_to_js_mappings b/tooling/bin/view_to_js_mappings deleted file mode 100755 index 483003aac5e..00000000000 --- a/tooling/bin/view_to_js_mappings +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require_relative '../lib/tooling/mappings/view_to_js_mappings' - -changes = ARGV.shift -output_file = ARGV.shift -changed_files = File.read(changes).split(' ') - -File.write(output_file, Tooling::Mappings::ViewToJsMappings.new.execute(changed_files).join(' ')) diff --git a/tooling/danger/product_intelligence.rb b/tooling/danger/analytics_instrumentation.rb index d25f966504f..ce5ee55e3ee 100644 --- a/tooling/danger/product_intelligence.rb +++ b/tooling/danger/analytics_instrumentation.rb @@ -3,13 +3,13 @@ module Tooling module Danger - module ProductIntelligence + module AnalyticsInstrumentation METRIC_DIRS = %w[lib/gitlab/usage/metrics/instrumentations ee/lib/gitlab/usage/metrics/instrumentations].freeze - APPROVED_LABEL = 'product intelligence::approved' - REVIEW_LABEL = 'product intelligence::review pending' + APPROVED_LABEL = 'analytics instrumentation::approved' + REVIEW_LABEL = 'analytics instrumentation::review pending' CHANGED_FILES_MESSAGE = <<~MSG - For the following files, a review from the [Data team and Product Intelligence team](https://gitlab.com/groups/gitlab-org/analytics-section/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) is recommended - Please check the ~"product intelligence" [Service Ping guide](https://docs.gitlab.com/ee/development/service_ping/) or the [Snowplow guide](https://docs.gitlab.com/ee/development/snowplow/). + For the following files, a review from the [Data team and Analytics Instrumentation team](https://gitlab.com/groups/gitlab-org/analytics-section/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) is recommended + Please check the ~"analytics instrumentation" [Service Ping guide](https://docs.gitlab.com/ee/development/service_ping/) or the [Snowplow guide](https://docs.gitlab.com/ee/development/snowplow/). For MR review guidelines, see the [Service Ping review guidelines](https://docs.gitlab.com/ee/development/service_ping/review_guidelines.html) or the [Snowplow review guidelines](https://docs.gitlab.com/ee/development/snowplow/review_guidelines.html). @@ -18,7 +18,7 @@ module Tooling MSG CHANGED_SCOPE_MESSAGE = <<~MSG - The following metrics could be affected by the modified scopes and require ~"product intelligence" review: + The following metrics could be affected by the modified scopes and require ~"analytics instrumentation" review: MSG @@ -33,13 +33,13 @@ module Tooling ].freeze def check! - # exit if not matching files or if no product intelligence labels - product_intelligence_paths_to_review = helper.changes_by_category[:product_intelligence] + analytics_instrumentation_paths_to_review = helper.changes.by_category(:analytics_instrumentation).files + labels_to_add = missing_labels - return if product_intelligence_paths_to_review.empty? || skip_review? + return if analytics_instrumentation_paths_to_review.empty? || skip_review? - warn format(CHANGED_FILES_MESSAGE, changed_files: helper.markdown_list(product_intelligence_paths_to_review)) unless has_approved_label? + warn format(CHANGED_FILES_MESSAGE, changed_files: helper.markdown_list(analytics_instrumentation_paths_to_review)) unless has_approved_label? helper.labels_to_add.concat(labels_to_add) unless labels_to_add.empty? end @@ -110,7 +110,7 @@ module Tooling return [] unless helper.ci? labels = [] - labels << 'product intelligence' unless helper.mr_has_labels?('product intelligence') + labels << 'analytics instrumentation' unless helper.mr_has_labels?('analytics instrumentation') labels << REVIEW_LABEL unless has_workflow_labels? labels diff --git a/tooling/danger/database_dictionary.rb b/tooling/danger/database_dictionary.rb new file mode 100644 index 00000000000..8776532ff84 --- /dev/null +++ b/tooling/danger/database_dictionary.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'yaml' + +module Tooling + module Danger + module DatabaseDictionary + DICTIONARY_PATH_REGEXP = %r{db/docs/.*\.yml} + + # `change_type` can be: + # - :added + # - :modified + # - :deleted + def database_dictionary_files(change_type:) + files = helper.public_send("#{change_type}_files") # rubocop:disable GitlabSecurity/PublicSend + + files.filter_map { |path| Found.new(path) if path =~ DICTIONARY_PATH_REGEXP } + end + + class Found + ATTRIBUTES = %w[ + table_name classes feature_categories description introduced_by_url milestone gitlab_schema + ].freeze + + attr_reader :path + + def initialize(path) + @path = path + end + + ATTRIBUTES.each do |attribute| + define_method(attribute) do + yaml[attribute] + end + end + + def raw + @raw ||= File.read(path) + end + + def ci_schema? + gitlab_schema == 'gitlab_ci' + end + + def main_schema? + gitlab_schema == 'gitlab_main' + end + + private + + def yaml + @yaml ||= YAML.safe_load(raw) + end + end + end + end +end diff --git a/tooling/danger/feature_flag.rb b/tooling/danger/feature_flag.rb index da0b7053af1..3fb20c561af 100644 --- a/tooling/danger/feature_flag.rb +++ b/tooling/danger/feature_flag.rb @@ -15,6 +15,12 @@ module Tooling files.select { |path| path =~ %r{\A(ee/)?config/feature_flags/} }.map { |path| Found.new(path) } end + # TODO: Move this to gitlab-dangerfiles helper + # https://gitlab.com/gitlab-org/ruby/gems/gitlab-dangerfiles/-/blob/master/lib/danger/plugins/internal/helper.rb + def stage_label + helper.mr_labels.find { |label| label.start_with?("devops::") } + end + class Found ATTRIBUTES = %w[name introduced_by_url rollout_issue_url milestone type group default_enabled].freeze diff --git a/tooling/danger/multiversion.rb b/tooling/danger/multiversion.rb new file mode 100644 index 00000000000..612881aaa1a --- /dev/null +++ b/tooling/danger/multiversion.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Tooling + module Danger + module Multiversion + FRONTEND_REGEX = %r{\A((ee|jh)/)?app/assets/.*(\.(vue|js|graphql))\z} + GRAPHQL_BACKEND_REGEX = %r{\A((ee|jh)/)?app/graphql/} + + def check! + return unless helper.ci? + return unless frontend_changed? && backend_changed? + + markdown <<~MARKDOWN + ## Multiversion compatibility + + This merge request updates GraphQL backend and frontend code. + + To prevent an incident, ensure the updated frontend code is backwards compatible. + + For more information, see the [multiversion compatibility documentation](https://docs.gitlab.com/ee/development/graphql_guide/reviewing.html#multiversion-compatibility). + MARKDOWN + end + + private + + def frontend_changed? + !git.modified_files.grep(FRONTEND_REGEX).empty? || !git.added_files.grep(FRONTEND_REGEX).empty? + end + + def backend_changed? + !git.added_files.grep(GRAPHQL_BACKEND_REGEX).empty? || !git.modified_files.grep(GRAPHQL_BACKEND_REGEX).empty? + end + end + end +end diff --git a/tooling/danger/project_helper.rb b/tooling/danger/project_helper.rb index 2a77ac337a2..5b5b998e3ea 100644 --- a/tooling/danger/project_helper.rb +++ b/tooling/danger/project_helper.rb @@ -26,11 +26,11 @@ module Tooling %r{\Adoc/api/graphql/reference/} => [:docs, :backend], %r{\Adoc/api/openapi/.*\.yaml\z} => [:docs, :backend], - [%r{usage_data\.rb}, %r{^(\+|-).*\s+(count|distinct_count|estimate_batch_distinct_count)\(.*\)(.*)$}] => [:database, :backend, :product_intelligence], + [%r{usage_data\.rb}, %r{^(\+|-).*\s+(count|distinct_count|estimate_batch_distinct_count)\(.*\)(.*)$}] => [:database, :backend, :analytics_instrumentation], %r{\A((ee|jh)/)?config/feature_flags/} => :feature_flag, - %r{doc/api/usage_data.md} => [:product_intelligence], + %r{doc/api/usage_data.md} => [:analytics_instrumentation], %r{\Adoc/.*(\.(md|png|gif|jpg|yml))\z} => :docs, %r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs, @@ -39,9 +39,9 @@ module Tooling %r{\Adata/deprecations/} => :none, %r{\Adata/removals/} => :none, - %r{\A((ee|jh)/)?app/finders/(.+/)?integrations/} => [:integrations_be, :database, :backend], - [%r{\A((ee|jh)/)?db/(geo/)?(migrate|post_migrate)/}, %r{(:integrations|:\w+_tracker_data)\b}] => [:integrations_be, :database, :migration], - [%r{\A((ee|jh)/)?(app|lib)/.+\.rb}, %r{\b(Integrations::|\.execute_(integrations|hooks))\b}] => [:integrations_be, :backend], + %r{\A((ee|jh)/)?app/finders/(.+/)?integrations/} => [:import_integrate_be, :database, :backend], + [%r{\A((ee|jh)/)?db/(geo/)?(migrate|post_migrate)/}, %r{(:integrations|:\w+_tracker_data)\b}] => [:import_integrate_be, :database, :migration], + [%r{\A((ee|jh)/)?(app|lib)/.+\.rb}, %r{\b(Integrations::|\.execute_(integrations|hooks))\b}] => [:import_integrate_be, :backend], %r{\A( ((ee|jh)/)?app/((?!.*clusters)(?!.*alert_management)(?!.*views)(?!.*assets).+/)?integration.+ | ((ee|jh)/)?app/((?!.*search).+/)?project_service.+ | @@ -53,20 +53,20 @@ module Tooling ((ee|jh)/)?lib/(.+/)?.*integration.+ | ((ee|jh)/)?lib/(.+/)?api/v3/github\.rb | ((ee|jh)/)?lib/(.+/)?api/github/entities\.rb - )\z}x => [:integrations_be, :backend], + )\z}x => [:import_integrate_be, :backend], %r{\A( ((ee|jh)/)?app/(views|assets)/((?!.*clusters)(?!.*alerts_settings).+/)?integration.+ | ((ee|jh)/)?app/(views|assets)/(.+/)?jira_connect.+ | ((ee|jh)/)?app/(views|assets)/((?!.*filtered_search).+/)?hooks?.+ - )\z}x => [:integrations_fe, :frontend], + )\z}x => [:import_integrate_fe, :frontend], %r{\A( app/assets/javascripts/tracking/.*\.js | spec/frontend/tracking/.*\.js | spec/frontend/tracking_spec\.js - )\z}x => [:frontend, :product_intelligence], - [%r{\.(vue|js)\z}, %r{trackRedis}] => [:frontend, :product_intelligence], + )\z}x => [:frontend, :analytics_instrumentation], + [%r{\.(vue|js)\z}, %r{trackRedis}] => [:frontend, :analytics_instrumentation], %r{\A((ee|jh)/)?app/assets/} => :frontend, %r{\A((ee|jh)/)?app/views/.*\.svg} => :frontend, %r{\A((ee|jh)/)?app/views/} => [:frontend, :backend], @@ -132,11 +132,11 @@ module Tooling %r{\A((ee|jh)/)?spec/support/shared_contexts/features/} => :test, %r{\A((ee|jh)/)?spec/support/helpers/features/} => :test, - %r{\A((spec/)?lib/generators/gitlab/usage_metric_)} => [:product_intelligence], - %r{\A((ee|jh)/)?lib/gitlab/usage_data_counters/.*\.yml\z} => [:product_intelligence], - %r{\A((ee|jh)/)?config/(events|metrics)/((.*\.yml)|(schema\.json))\z} => [:product_intelligence], - %r{\A((ee|jh)/)?lib/gitlab/usage_data(_counters)?(/|\.rb)} => [:backend, :product_intelligence], - %r{\A((ee|jh)/)?(spec/)?lib/gitlab/usage(/|\.rb)} => [:backend, :product_intelligence], + %r{\A((spec/)?lib/generators/gitlab/usage_metric_)} => [:analytics_instrumentation], + %r{\A((ee|jh)/)?lib/gitlab/usage_data_counters/.*\.yml\z} => [:analytics_instrumentation], + %r{\A((ee|jh)/)?config/(events|metrics)/((.*\.yml)|(schema\.json))\z} => [:analytics_instrumentation], + %r{\A((ee|jh)/)?lib/gitlab/usage_data(_counters)?(/|\.rb)} => [:backend, :analytics_instrumentation], + %r{\A((ee|jh)/)?(spec/)?lib/gitlab/usage(/|\.rb)} => [:backend, :analytics_instrumentation], %r{\A( lib/gitlab/tracking\.rb | spec/lib/gitlab/tracking_spec\.rb | @@ -146,11 +146,11 @@ module Tooling (spec/)?lib/generators/gitlab/usage_metric_definition/redis_hll_generator(_spec)?\.rb | lib/generators/rails/usage_metric_definition_generator\.rb | spec/lib/generators/usage_metric_definition_generator_spec\.rb | - generator_templates/usage_metric_definition/metric_definition\.yml)\z}x => [:backend, :product_intelligence], - %r{gitlab/usage_data(_spec)?\.rb} => [:product_intelligence], - [%r{\.haml\z}, %r{data: \{ track}] => [:product_intelligence], - [%r{\.(rb|haml)\z}, %r{Gitlab::Tracking\.(event|enabled\?|options)$}] => [:product_intelligence], - [%r{\.(vue|js)\z}, %r{(Tracking.event|/\btrack\(/|data-track-action)}] => [:product_intelligence], + generator_templates/usage_metric_definition/metric_definition\.yml)\z}x => [:backend, :analytics_instrumentation], + %r{gitlab/usage_data(_spec)?\.rb} => [:analytics_instrumentation], + [%r{\.haml\z}, %r{data: \{ track}] => [:analytics_instrumentation], + [%r{\.(rb|haml)\z}, %r{Gitlab::Tracking\.(event|enabled\?|options)$}] => [:analytics_instrumentation], + [%r{\.(vue|js)\z}, %r{(Tracking.event|/\btrack\(/|data-track-action)}] => [:analytics_instrumentation], %r{\A((ee|jh)/)?app/(?!assets|views)[^/]+} => :backend, %r{\A((ee|jh)/)?(bin|config|generator_templates|lib|rubocop)/} => :backend, diff --git a/tooling/danger/sidekiq_args.rb b/tooling/danger/sidekiq_args.rb new file mode 100644 index 00000000000..d06bb92ca6d --- /dev/null +++ b/tooling/danger/sidekiq_args.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Tooling + module Danger + module SidekiqArgs + include ::Danger::Helpers + + WORKER_FILES_REGEX = 'app/workers' + EE_PREFIX = 'ee/' + DEF_PERFORM = "def perform" + DEF_PERFORM_REGEX = /[\s+-]*def perform\((.*)\)/ + BEFORE_DEF_PERFORM_REGEX = /^[\s-]*def perform\b/ + AFTER_DEF_PERFORM_REGEX = /^[\s+]*def perform\b/ + + SUGGEST_MR_COMMENT = <<~SUGGEST_COMMENT + Please follow the [sidekiq development guidelines](https://docs.gitlab.com/ee/development/sidekiq/compatibility_across_updates.html#changing-the-arguments-for-a-worker) when changing sidekiq worker arguments. + SUGGEST_COMMENT + + def changed_worker_files(ee: :include) + changed_files = helper.all_changed_files + folder_prefix = + case ee + when :include + "(#{EE_PREFIX})?" + when :only + EE_PREFIX + when :exclude + nil + end + + changed_files.grep(%r{\A#{folder_prefix}#{WORKER_FILES_REGEX}}) + end + + def args_changed?(diff) + # Find the "before" and "after" versions of the perform method definition + before_def_perform = diff.find { |line| BEFORE_DEF_PERFORM_REGEX.match?(line) } + after_def_perform = diff.find { |line| AFTER_DEF_PERFORM_REGEX.match?(line) } + + # args are not changed if there is no before or after def perform method + return false unless before_def_perform && after_def_perform + + # Extract the perform method arguments from the "before" and "after" versions + before_args, after_args = diff.flat_map { |line| line.scan(DEF_PERFORM_REGEX) } + + before_args != after_args + end + + def add_comment_for_matched_line(filename) + diff = helper.changed_lines(filename) + return unless args_changed?(diff) + + file_lines = project_helper.file_lines(filename) + + perform_method_line = file_lines.index { |line| line.include?(DEF_PERFORM) } + markdown(format(SUGGEST_MR_COMMENT), file: filename, line: perform_method_line.succ) + end + end + end +end diff --git a/tooling/danger/sidekiq_queues.rb b/tooling/danger/sidekiq_queues.rb index ae32b128682..bd6480fcba6 100644 --- a/tooling/danger/sidekiq_queues.rb +++ b/tooling/danger/sidekiq_queues.rb @@ -14,7 +14,7 @@ module Tooling def changed_queue_names @changed_queue_names ||= (new_queues.values_at(*old_queues.keys) - old_queues.values) - .compact.map { |queue| queue[:name] } + .compact.map { |queue| queue[:name] } # rubocop:disable Rails/Pluck end private diff --git a/tooling/danger/specs.rb b/tooling/danger/specs.rb index f95a798d53e..68cab141656 100644 --- a/tooling/danger/specs.rb +++ b/tooling/danger/specs.rb @@ -1,55 +1,19 @@ # frozen_string_literal: true -require_relative 'suggestor' +Dir[File.expand_path('specs/*_suggestion.rb', __dir__)].each { |file| require file } module Tooling module Danger module Specs - include ::Tooling::Danger::Suggestor - SPEC_FILES_REGEX = 'spec/' EE_PREFIX = 'ee/' - PROJECT_FACTORIES = %w[ - :project - :project_empty_repo - :forked_project_with_submodules - :project_with_design + SUGGESTIONS = [ + FeatureCategorySuggestion, + MatchWithArraySuggestion, + ProjectFactorySuggestion ].freeze - PROJECT_FACTORY_REGEX = / - ^\+? # Start of the line, which may or may not have a `+` - (?<head>\s*) # 0-many leading whitespace captured in a group named head - let!? # Literal `let` which may or may not end in `!` - (?<tail> # capture group named tail - \([^)]+\) # Two parenthesis with any non-parenthesis characters between them - \s*\{\s* # Opening curly brace surrounded by 0-many whitespace characters - create\( # literal - (?:#{PROJECT_FACTORIES.join('|')}) # Any of the project factory names - \W # Non-word character, avoid matching factories like :project_authorization - ) # end capture group named tail - /x.freeze - - PROJECT_FACTORY_REPLACEMENT = '\k<head>let_it_be\k<tail>' - PROJECT_FACTORY_SUGGESTION = <<~SUGGEST_COMMENT - Project creations are very slow. Use `let_it_be`, `build` or `build_stubbed` if possible. - See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#optimize-factory-usage) - for background information and alternative options. - SUGGEST_COMMENT - - MATCH_WITH_ARRAY_REGEX = /(?<to>to\(?\s*)(?<matcher>match|eq)(?<expectation>[( ]?\[(?=.*,)[^\]]+)/.freeze - MATCH_WITH_ARRAY_REPLACEMENT = '\k<to>match_array\k<expectation>' - MATCH_WITH_ARRAY_SUGGESTION = <<~SUGGEST_COMMENT - If order of the result is not important, please consider using `match_array` to avoid flakiness. - SUGGEST_COMMENT - - RSPEC_TOP_LEVEL_DESCRIBE_REGEX = /^\+.?RSpec\.describe(.+)/.freeze - FEATURE_CATEGORY_SUGGESTION = <<~SUGGESTION_MARKDOWN - Consider adding `feature_category: <feature_category_name>` for this example if it is not set already. - See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#feature-category-metadata). - SUGGESTION_MARKDOWN - FEATURE_CATEGORY_KEYWORD = 'feature_category' - def changed_specs_files(ee: :include) changed_files = helper.all_changed_files folder_prefix = @@ -65,43 +29,9 @@ module Tooling changed_files.grep(%r{\A#{folder_prefix}#{SPEC_FILES_REGEX}}) end - def add_suggestions_for_match_with_array(filename) - add_suggestion( - filename: filename, - regex: MATCH_WITH_ARRAY_REGEX, - replacement: MATCH_WITH_ARRAY_REPLACEMENT, - comment_text: MATCH_WITH_ARRAY_SUGGESTION - ) - end - - def add_suggestions_for_project_factory_usage(filename) - add_suggestion( - filename: filename, - regex: PROJECT_FACTORY_REGEX, - replacement: PROJECT_FACTORY_REPLACEMENT, - comment_text: PROJECT_FACTORY_SUGGESTION - ) - end - - def add_suggestions_for_feature_category(filename) - file_lines = project_helper.file_lines(filename) - changed_lines = helper.changed_lines(filename) - - changed_lines.each_with_index do |changed_line, i| - next unless changed_line =~ RSPEC_TOP_LEVEL_DESCRIBE_REGEX - - next_line_in_file = file_lines[file_lines.find_index(changed_line.delete_prefix('+')) + 1] - - if changed_line.include?(FEATURE_CATEGORY_KEYWORD) || next_line_in_file.to_s.include?(FEATURE_CATEGORY_KEYWORD) - next - end - - line_number = file_lines.find_index(changed_line.delete_prefix('+')) - next unless line_number - - suggested_line = file_lines[line_number] - - markdown(comment(FEATURE_CATEGORY_SUGGESTION, suggested_line), file: filename, line: line_number.succ) + def add_suggestions_for(filename) + SUGGESTIONS.each do |suggestion| + suggestion.new(filename, context: self).suggest end end end diff --git a/tooling/danger/specs/feature_category_suggestion.rb b/tooling/danger/specs/feature_category_suggestion.rb new file mode 100644 index 00000000000..5acf73c8956 --- /dev/null +++ b/tooling/danger/specs/feature_category_suggestion.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require_relative '../suggestion' + +module Tooling + module Danger + module Specs + class FeatureCategorySuggestion < Suggestion + RSPEC_TOP_LEVEL_DESCRIBE_REGEX = /^\+.?RSpec\.describe(.+)/ + SUGGESTION = <<~SUGGESTION_MARKDOWN + Consider adding `feature_category: <feature_category_name>` for this example if it is not set already. + See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#feature-category-metadata). + SUGGESTION_MARKDOWN + FEATURE_CATEGORY_KEYWORD = 'feature_category' + + def suggest + file_lines = project_helper.file_lines(filename) + changed_lines = helper.changed_lines(filename) + + changed_lines.each do |changed_line| + next unless changed_line =~ RSPEC_TOP_LEVEL_DESCRIBE_REGEX + + line_number = file_lines.find_index(changed_line.delete_prefix('+')) + next unless line_number + + # Get the top level RSpec.describe line and the next 5 lines + lines_to_check = file_lines[line_number, 5] + # Remove all the lines after the first one that ends in `do` + last_line_number_of_describe_declaration = lines_to_check.index { |line| line.end_with?(' do') } + lines_to_check = lines_to_check[0..last_line_number_of_describe_declaration] + + next if lines_to_check.any? { |line| line.include?(FEATURE_CATEGORY_KEYWORD) } + + suggested_line = file_lines[line_number] + + markdown(comment(SUGGESTION, suggested_line), file: filename, line: line_number.succ) + end + end + end + end + end +end diff --git a/tooling/danger/specs/match_with_array_suggestion.rb b/tooling/danger/specs/match_with_array_suggestion.rb new file mode 100644 index 00000000000..eb0f7a1a832 --- /dev/null +++ b/tooling/danger/specs/match_with_array_suggestion.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require_relative '../suggestion' + +module Tooling + module Danger + module Specs + class MatchWithArraySuggestion < Suggestion + MATCH = /(?<to>to\(?\s*)(?<matcher>match|eq)(?<expectation>[( ]?\[(?=.*,)[^\]]+)/ + REPLACEMENT = '\k<to>match_array\k<expectation>' + SUGGESTION = <<~SUGGEST_COMMENT + If order of the result is not important, please consider using `match_array` to avoid flakiness. + SUGGEST_COMMENT + end + end + end +end diff --git a/tooling/danger/specs/project_factory_suggestion.rb b/tooling/danger/specs/project_factory_suggestion.rb new file mode 100644 index 00000000000..78230db9585 --- /dev/null +++ b/tooling/danger/specs/project_factory_suggestion.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require_relative '../suggestion' + +module Tooling + module Danger + module Specs + class ProjectFactorySuggestion < Suggestion + PROJECT_FACTORIES = %w[ + :project + :project_empty_repo + :forked_project_with_submodules + :project_with_design + ].freeze + + MATCH = / + ^\+? # Start of the line, which may or may not have a `+` + (?<head>\s*) # 0-many leading whitespace captured in a group named head + let!? # Literal `let` which may or may not end in `!` + (?<tail> # capture group named tail + \([^)]+\) # Two parenthesis with any non-parenthesis characters between them + \s*\{\s* # Opening curly brace surrounded by 0-many whitespace characters + create\( # literal + (?:#{PROJECT_FACTORIES.join('|')}) # Any of the project factory names + \W # Non-word character, avoid matching factories like :project_badge + ) # end capture group named tail + /x + + REPLACEMENT = '\k<head>let_it_be\k<tail>' + SUGGESTION = <<~SUGGEST_COMMENT + Project creations are very slow. Using `let_it_be`, `build` or `build_stubbed` can improve test performance. + + Warning: `let_it_be` may not be suitable if your test modifies data as this could result in state leaks! + + In those cases, please use `let_it_be_with_reload` or `let_it_be_with_refind` instead. + + If your are unsure which is the right method to use, + please refer to [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#optimize-factory-usage) + for background information and alternative options for optimizing factory usage. + + Feel free to ignore this comment if you know `let` or `let!` are the better options and/or worry about causing state leaks. + SUGGEST_COMMENT + end + end + end +end diff --git a/tooling/danger/stable_branch.rb b/tooling/danger/stable_branch.rb index 9b467146096..bba198d1310 100644 --- a/tooling/danger/stable_branch.rb +++ b/tooling/danger/stable_branch.rb @@ -42,18 +42,18 @@ module Tooling MSG PIPELINE_EXPEDITE_ERROR_MESSAGE = <<~MSG - ~"pipeline:expedite" is not allowed on stable branches because it causes the `e2e:package-and-test` job to be skipped. + ~"pipeline:expedite" is not allowed on stable branches because it causes the `e2e:package-and-test-ee` job to be skipped. MSG NEEDS_PACKAGE_AND_TEST_MESSAGE = <<~MSG - The `e2e:package-and-test` job is not present, has been canceled, or needs to be automatically triggered. + The `e2e:package-and-test-ee` job is not present, has been canceled, or needs to be automatically triggered. Please ensure the job is present in the latest pipeline, if necessary, retry the `danger-review` job. - Read the "QA e2e:package-and-test" section for more details. + Read the "QA e2e:package-and-test-ee" section for more details. MSG WARN_PACKAGE_AND_TEST_MESSAGE = <<~MSG - **The `e2e:package-and-test` job needs to succeed or have approval from a Software Engineer in Test.** - Read the "QA e2e:package-and-test" section for more details. + **The `e2e:package-and-test-ee` job needs to succeed or have approval from a Software Engineer in Test.** + Read the "QA e2e:package-and-test-ee" section for more details. MSG # rubocop:disable Style/SignalException @@ -74,7 +74,7 @@ module Tooling if status.nil? || FAILING_PACKAGE_AND_TEST_STATUSES.include?(status) # rubocop:disable Style/GuardClause fail NEEDS_PACKAGE_AND_TEST_MESSAGE else - warn WARN_PACKAGE_AND_TEST_MESSAGE unless status == 'success' + warn WARN_PACKAGE_AND_TEST_MESSAGE end end # rubocop:enable Style/SignalException @@ -85,12 +85,12 @@ module Tooling !has_flaky_failure_label? end - private - def valid_stable_branch? !!stable_target_branch && !helper.security_mr? end + private + def package_and_test_bridge_and_pipeline_status mr_head_pipeline_id = gitlab.mr_json.dig('head_pipeline', 'id') return unless mr_head_pipeline_id @@ -110,7 +110,7 @@ module Tooling gitlab .api .pipeline_bridges(helper.mr_target_project_id, mr_head_pipeline_id) - &.find { |bridge| bridge['name'] == 'e2e:package-and-test' } + &.find { |bridge| bridge['name'].include?('package-and-test-ee') } end def stable_target_branch diff --git a/tooling/danger/suggestion.rb b/tooling/danger/suggestion.rb new file mode 100644 index 00000000000..da3c6b0e76f --- /dev/null +++ b/tooling/danger/suggestion.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'forwardable' +require_relative 'suggestor' + +module Tooling + module Danger + # A basic suggestion. + # + # A subclass needs to define the following constants: + # * MATCH (Regexp) - A Regexp to match file lines + # * REPLACEMENT (String) - A suggestion replacement text + # * SUGGESTION (String) - A suggestion text + # + # @see Suggestor + class Suggestion + extend Forwardable + include ::Tooling::Danger::Suggestor + + def_delegators :@context, :helper, :project_helper, :markdown + + attr_reader :filename + + def initialize(filename, context:) + @filename = filename + @context = context + end + + def suggest + add_suggestion( + filename: filename, + regex: self.class::MATCH, + replacement: self.class::REPLACEMENT, + comment_text: self.class::SUGGESTION + ) + end + end + end +end diff --git a/tooling/docs/deprecation_handling.rb b/tooling/docs/deprecation_handling.rb index bcdf73e0044..5996a0c89c1 100644 --- a/tooling/docs/deprecation_handling.rb +++ b/tooling/docs/deprecation_handling.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'erb' module Docs @@ -7,11 +8,7 @@ module Docs @type = type @yaml_glob_path = Rails.root.join("data/#{type.pluralize}/**/*.yml") @template_path = Rails.root.join("data/#{type.pluralize}/templates/_#{type}_template.md.erb") - @milestone_key_name = if type == "deprecation" - "announcement_milestone" - else - "removal_milestone" - end + @milestone_key_name = "removal_milestone" end def render diff --git a/tooling/graphql/docs/helper.rb b/tooling/graphql/docs/helper.rb index f25e69a1e2f..bd8ff0cf862 100644 --- a/tooling/graphql/docs/helper.rb +++ b/tooling/graphql/docs/helper.rb @@ -56,7 +56,7 @@ module Tooling <<-MD.strip_heredoc --- stage: Manage - group: Integrations + group: Import and Integrate info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments --- diff --git a/tooling/lib/tooling/fast_quarantine.rb b/tooling/lib/tooling/fast_quarantine.rb new file mode 100644 index 00000000000..a0dc8bc460b --- /dev/null +++ b/tooling/lib/tooling/fast_quarantine.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Tooling + class FastQuarantine + def initialize(fast_quarantine_path:) + warn "#{fast_quarantine_path} doesn't exist!" unless File.exist?(fast_quarantine_path.to_s) + + @fast_quarantine_path = fast_quarantine_path + end + + def identifiers + @identifiers ||= begin + quarantined_entity_identifiers = File.read(fast_quarantine_path).lines + quarantined_entity_identifiers.compact! + quarantined_entity_identifiers.map! do |quarantined_entity_identifier| + quarantined_entity_identifier.delete_prefix('./').strip + end + rescue => e # rubocop:disable Style/RescueStandardError + $stdout.puts e + [] + end + end + + def skip_example?(example) + identifiers.find do |quarantined_entity_identifier| + case quarantined_entity_identifier + when /^.+_spec\.rb\[[\d:]+\]$/ # example id, e.g. spec/tasks/gitlab/usage_data_rake_spec.rb[1:5:2:1] + example.id == "./#{quarantined_entity_identifier}" + when /^.+_spec\.rb:\d+$/ # file + line, e.g. spec/tasks/gitlab/usage_data_rake_spec.rb:42 + fetch_metadata_from_ancestors(example, :location) + .any?("./#{quarantined_entity_identifier}") + when /^.+_spec\.rb$/ # whole file, e.g. ee/spec/features/boards/swimlanes/epics_swimlanes_sidebar_spec.rb + fetch_metadata_from_ancestors(example, :file_path) + .any?("./#{quarantined_entity_identifier}") + end + end + end + + private + + attr_reader :fast_quarantine_path + + def fetch_metadata_from_ancestors(example, attribute) + metadata = [example.metadata[attribute]] + example_group = example.metadata[:example_group] + + loop do + break if example_group.nil? + + metadata << example_group[attribute] + example_group = example_group[:parent_example_group] + end + + metadata + end + end +end diff --git a/tooling/lib/tooling/find_changes.rb b/tooling/lib/tooling/find_changes.rb new file mode 100755 index 00000000000..c498c83d24b --- /dev/null +++ b/tooling/lib/tooling/find_changes.rb @@ -0,0 +1,109 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'gitlab' +require_relative 'helpers/predictive_tests_helper' + +module Tooling + class FindChanges + include Helpers::PredictiveTestsHelper + + ALLOWED_FILE_TYPES = ['.js', '.vue', '.md', '.scss'].freeze + + def initialize( + from:, + changed_files_pathname: nil, + predictive_tests_pathname: nil, + frontend_fixtures_mapping_pathname: nil + ) + + raise ArgumentError, ':from can only be :api or :changed_files' unless + %i[api changed_files].include?(from) + + @gitlab_token = ENV['PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE'] || '' + @gitlab_endpoint = ENV['CI_API_V4_URL'] + @mr_project_path = ENV['CI_MERGE_REQUEST_PROJECT_PATH'] + @mr_iid = ENV['CI_MERGE_REQUEST_IID'] + @changed_files_pathname = changed_files_pathname + @predictive_tests_pathname = predictive_tests_pathname + @frontend_fixtures_mapping_pathname = frontend_fixtures_mapping_pathname + @from = from + end + + def execute + if changed_files_pathname.nil? + raise ArgumentError, "A path to the changed files file must be given as :changed_files_pathname" + end + + case @from + when :api + write_array_to_file(changed_files_pathname, file_changes + frontend_fixture_files, append: false) + else + write_array_to_file(changed_files_pathname, frontend_fixture_files, append: true) + end + end + + def only_allowed_files_changed + file_changes.any? && file_changes.all? { |file| ALLOWED_FILE_TYPES.include?(File.extname(file)) } + end + + private + + attr_reader :gitlab_token, :gitlab_endpoint, :mr_project_path, + :mr_iid, :changed_files_pathname, :predictive_tests_pathname, :frontend_fixtures_mapping_pathname + + def gitlab + @gitlab ||= begin + Gitlab.configure do |config| + config.endpoint = gitlab_endpoint + config.private_token = gitlab_token + end + + Gitlab + end + end + + def add_frontend_fixture_files? + predictive_tests_pathname && frontend_fixtures_mapping_pathname + end + + def frontend_fixture_files + # If we have a `test file -> JSON frontend fixture` mapping file, we add the files JSON frontend fixtures + # files to the list of changed files so that Jest can automatically run the dependent tests + # using --findRelatedTests flag. + empty = [].freeze + + test_files.flat_map do |test_file| + frontend_fixtures_mapping[test_file] || empty + end + end + + def file_changes + @file_changes ||= + case @from + when :api + mr_changes.changes.flat_map do |change| + change.to_h.values_at('old_path', 'new_path') + end.uniq + else + read_array_from_file(changed_files_pathname) + end + end + + def mr_changes + @mr_changes ||= gitlab.merge_request_changes(mr_project_path, mr_iid) + end + + def test_files + return [] if !predictive_tests_pathname || !File.exist?(predictive_tests_pathname) + + read_array_from_file(predictive_tests_pathname) + end + + def frontend_fixtures_mapping + return {} if !frontend_fixtures_mapping_pathname || !File.exist?(frontend_fixtures_mapping_pathname) + + JSON.parse(File.read(frontend_fixtures_mapping_pathname)) # rubocop:disable Gitlab/Json + end + end +end diff --git a/tooling/lib/tooling/find_codeowners.rb b/tooling/lib/tooling/find_codeowners.rb index cc37d4db1ec..e542ab9967c 100644 --- a/tooling/lib/tooling/find_codeowners.rb +++ b/tooling/lib/tooling/find_codeowners.rb @@ -48,11 +48,7 @@ module Tooling def load_config config_path = "#{__dir__}/../../config/CODEOWNERS.yml" - if YAML.respond_to?(:safe_load_file) # Ruby 3.0+ - YAML.safe_load_file(config_path, symbolize_names: true) - else - YAML.safe_load(File.read(config_path), symbolize_names: true) - end + YAML.safe_load_file(config_path, symbolize_names: true) end # Copied and modified from ee/lib/gitlab/code_owners/file.rb diff --git a/tooling/lib/tooling/find_files_using_feature_flags.rb b/tooling/lib/tooling/find_files_using_feature_flags.rb new file mode 100644 index 00000000000..27cace60408 --- /dev/null +++ b/tooling/lib/tooling/find_files_using_feature_flags.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require 'test_file_finder' +require_relative 'helpers/predictive_tests_helper' + +module Tooling + class FindFilesUsingFeatureFlags + include Helpers::PredictiveTestsHelper + + def initialize(changed_files_pathname:, feature_flags_base_folder: 'config/feature_flags') + @changed_files_pathname = changed_files_pathname + @changed_files = read_array_from_file(changed_files_pathname) + @feature_flags_base_folders = folders_for_available_editions(feature_flags_base_folder) + end + + def execute + ff_union_regexp = Regexp.union(feature_flag_filenames) + + files_using_modified_feature_flags = ruby_files.select do |ruby_file| + ruby_file if ff_union_regexp.match?(File.read(ruby_file)) + end + + write_array_to_file(changed_files_pathname, files_using_modified_feature_flags.uniq) + end + + def filter_files + @_filter_files ||= changed_files.select do |filename| + filename.start_with?(*feature_flags_base_folders) && + File.basename(filename).end_with?('.yml') && + File.exist?(filename) + end + end + + private + + def feature_flag_filenames + filter_files.map do |feature_flag_pathname| + File.basename(feature_flag_pathname).delete_suffix('.yml') + end + end + + def ruby_files + Dir["**/*.rb"].reject { |pathname| pathname.start_with?('vendor') } + end + + attr_reader :changed_files, :changed_files_pathname, :feature_flags_base_folders + end +end diff --git a/tooling/lib/tooling/find_tests.rb b/tooling/lib/tooling/find_tests.rb new file mode 100644 index 00000000000..bf7a608878b --- /dev/null +++ b/tooling/lib/tooling/find_tests.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'test_file_finder' +require_relative 'helpers/predictive_tests_helper' + +module Tooling + class FindTests + include Helpers::PredictiveTestsHelper + + def initialize(changed_files_pathname, predictive_tests_pathname) + @predictive_tests_pathname = predictive_tests_pathname + @changed_files = read_array_from_file(changed_files_pathname) + end + + def execute + tff = TestFileFinder::FileFinder.new(paths: changed_files).tap do |file_finder| + file_finder.use TestFileFinder::MappingStrategies::PatternMatching.load('tests.yml') + + if ENV['RSPEC_TESTS_MAPPING_ENABLED'] == 'true' + file_finder.use TestFileFinder::MappingStrategies::DirectMatching.load_json(ENV['RSPEC_TESTS_MAPPING_PATH']) + end + end + + write_array_to_file(predictive_tests_pathname, tff.test_files.uniq) + end + + private + + attr_reader :changed_files, :matching_tests, :predictive_tests_pathname + end +end diff --git a/tooling/lib/tooling/gettext_extractor.rb b/tooling/lib/tooling/gettext_extractor.rb new file mode 100644 index 00000000000..3cc05c80a2d --- /dev/null +++ b/tooling/lib/tooling/gettext_extractor.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require 'parallel' +require 'gettext/po' +require 'gettext/po_entry' +require 'gettext/tools/xgettext' +require 'gettext/tools/parser/erb' +require 'gettext/tools/parser/ruby' +require 'json' +require 'open3' + +module Tooling + class GettextExtractor < GetText::Tools::XGetText + class HamlParser < GetText::RubyParser + require 'hamlit' + def parse_source(source) + super(Hamlit::Engine.new.call(source)) + end + end + + def initialize( + backend_glob: "{ee/,}{app,lib,config,locale}/**/*.{rb,erb,haml}", + glob_base: nil, + package_name: 'gitlab', + package_version: '1.0.0' + ) + super() + @backend_glob = backend_glob + @package_name = package_name + @glob_base = glob_base || Dir.pwd + @package_version = package_version + # Ensure that the messages are ordered by id + @po_order = :msgid + @po_format_options = { + # No line breaks within a message + max_line_width: -1, + # Do not print references to files + include_reference_comment: false + } + end + + def parse(_paths) + po = GetText::PO.new + parse_backend_files.each do |po_entry| + merge_po_entries(po, po_entry) + end + parse_frontend_files.each do |po_entry| + merge_po_entries(po, po_entry) + end + po + end + + # Overrides method from GetText::Tools::XGetText + # This makes a method public and passes in an empty array of paths, + # as our overidden "parse" method needs no paths + def generate_pot + super([]) + end + + private + + # Overrides method from GetText::Tools::XGetText + # in order to remove revision dates, as we check in our locale/gitlab.pot + def header_content + super.gsub(/^POT?-(?:Creation|Revision)-Date:.*\n/, '') + end + + def merge_po_entries(po, po_entry) + existing_entry = po[po_entry.msgctxt, po_entry.msgid] + po_entry = existing_entry.merge(po_entry) if existing_entry + + po[po_entry.msgctxt, po_entry.msgid] = po_entry + end + + def parse_backend_file(path) + source = ::File.read(path) + # Do not bother parsing files not containing `_(` + # All of our translation helpers, _(, s_(), N_(), etc. + # contain it. So we can skip parsing files not containing it + return [] unless source.include?('_(') + + case ::File.extname(path) + when '.rb' + GetText::RubyParser.new(path).parse_source(source) + when '.haml' + HamlParser.new(path).parse_source(source) + when '.erb' + GetText::ErbParser.new(path).parse + else + raise NotImplementedError + end + end + + def parse_backend_files + files = Dir.glob(File.join(@glob_base, @backend_glob)) + Parallel.flat_map(files) { |item| parse_backend_file(item) } + end + + def parse_frontend_files + results, status = Open3.capture2('node scripts/frontend/extract_gettext_all.js --all') + raise StandardError, "Could not parse frontend files" unless status.success? + + # rubocop:disable Gitlab/Json + JSON.parse(results) + .values + .flatten(1) + .collect { |entry| create_po_entry(*entry) } + # rubocop:enable Gitlab/Json + end + end +end diff --git a/tooling/lib/tooling/helpers/file_handler.rb b/tooling/lib/tooling/helpers/file_handler.rb new file mode 100644 index 00000000000..88248e31df2 --- /dev/null +++ b/tooling/lib/tooling/helpers/file_handler.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'fileutils' + +module Tooling + module Helpers + module FileHandler + def read_array_from_file(file) + FileUtils.touch file + + File.read(file).split(' ') + end + + def write_array_to_file(file, content_array, append: true) + FileUtils.touch file + + # We sort the array to make it easier to read the output file + content_array.sort! + + output_content = + if append + [File.read(file), *content_array].join(' ').lstrip + else + content_array.join(' ') + end + + File.write(file, output_content) + end + end + end +end diff --git a/tooling/lib/tooling/mappings/base.rb b/tooling/lib/tooling/helpers/predictive_tests_helper.rb index 93d3a967114..b8e5a30024e 100644 --- a/tooling/lib/tooling/mappings/base.rb +++ b/tooling/lib/tooling/helpers/predictive_tests_helper.rb @@ -1,22 +1,13 @@ # frozen_string_literal: true require_relative '../../../../lib/gitlab_edition' +require_relative '../helpers/file_handler' # Returns system specs files that are related to the JS files that were changed in the MR. module Tooling - module Mappings - class Base - # Input: A list of space-separated files - # Output: A list of space-separated specs files (JS, Ruby, ...) - def execute(changed_files) - raise "Not Implemented" - end - - # Input: A list of space-separated files - # Output: array/hash of files - def filter_files(changed_files) - raise "Not Implemented" - end + module Helpers + module PredictiveTestsHelper + include FileHandler # Input: A folder # Output: An array of folders, each prefixed with a GitLab edition diff --git a/tooling/lib/tooling/kubernetes_client.rb b/tooling/lib/tooling/kubernetes_client.rb index ab914db5777..5579f130a84 100644 --- a/tooling/lib/tooling/kubernetes_client.rb +++ b/tooling/lib/tooling/kubernetes_client.rb @@ -6,72 +6,23 @@ require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen) module Tooling class KubernetesClient - RESOURCE_LIST = 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd' K8S_ALLOWED_NAMESPACES_REGEX = /^review-(?!apps).+/.freeze CommandFailedError = Class.new(StandardError) - attr_reader :namespace + def cleanup_namespaces_by_created_at(created_before:) + stale_namespaces = namespaces_created_before(created_before: created_before) - def initialize(namespace:) - @namespace = namespace - end - - def cleanup_by_release(release_name:, wait: true) - delete_by_selector(release_name: release_name, wait: wait) - delete_by_matching_name(release_name: release_name) - end - - def cleanup_by_created_at(resource_type:, created_before:, wait: true) - resource_names = resource_names_created_before(resource_type: resource_type, created_before: created_before) - return if resource_names.empty? - - delete_by_exact_names(resource_type: resource_type, resource_names: resource_names, wait: wait) - end - - def cleanup_review_app_namespaces(created_before:, wait: true) - namespaces = review_app_namespaces_created_before(created_before: created_before) - return if namespaces.empty? - - delete_namespaces_by_exact_names(resource_names: namespaces, wait: wait) - end - - private - - def delete_by_selector(release_name:, wait:) - selector = case release_name - when String - %(-l release="#{release_name}") - when Array - %(-l 'release in (#{release_name.join(', ')})') - else - raise ArgumentError, 'release_name must be a string or an array' - end - - command = [ - 'delete', - RESOURCE_LIST, - %(--namespace "#{namespace}"), - '--now', - '--ignore-not-found', - %(--wait=#{wait}), - selector - ] + # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby. + review_apps_stale_namespaces = stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) } + return if review_apps_stale_namespaces.empty? - run_command(command) + delete_namespaces(review_apps_stale_namespaces) end - def delete_by_exact_names(resource_names:, wait:, resource_type: nil) - command = [ - 'delete', - resource_type, - %(--namespace "#{namespace}"), - '--now', - '--ignore-not-found', - %(--wait=#{wait}), - resource_names.join(' ') - ] + def delete_namespaces(namespaces) + return if namespaces.any? { |ns| !K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) } - run_command(command) + run_command("kubectl delete namespace --now --ignore-not-found #{namespaces.join(' ')}") end def delete_namespaces_by_exact_names(resource_names:, wait:) @@ -87,87 +38,29 @@ module Tooling run_command(command) end - def delete_by_matching_name(release_name:) - resource_names = raw_resource_names - command = [ - 'delete', - %(--namespace "#{namespace}"), - '--ignore-not-found' - ] - - Array(release_name).each do |release| - resource_names - .select { |resource_name| resource_name.include?(release) } - .each { |matching_resource| run_command(command + [matching_resource]) } - end - end - - def raw_resource_names - command = [ - 'get', - RESOURCE_LIST, - %(--namespace "#{namespace}"), - '-o name' - ] - run_command(command).lines.map(&:strip) - end - - def resource_names_created_before(resource_type:, created_before:) - command = [ - 'get', - resource_type, - %(--namespace "#{namespace}"), - "--sort-by='{.metadata.creationTimestamp}'", - '-o json' - ] + def namespaces_created_before(created_before:) + response = run_command("kubectl get namespace --all-namespaces --sort-by='{.metadata.creationTimestamp}' -o json") - response = run_command(command) - - resources_created_before_date(response, created_before) - end - - def review_app_namespaces_created_before(created_before:) - command = [ - 'get', - 'namespace', - "--sort-by='{.metadata.creationTimestamp}'", - '-o json' - ] - - response = run_command(command) - - stale_namespaces = resources_created_before_date(response, created_before) - - # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby. - stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) } - end - - def resources_created_before_date(response, date) items = JSON.parse(response)['items'] # rubocop:disable Gitlab/Json - - items.each_with_object([]) do |item, result| + items.filter_map do |item| item_created_at = Time.parse(item.dig('metadata', 'creationTimestamp')) - if item_created_at < date - resource_name = item.dig('metadata', 'name') - result << resource_name - end + item.dig('metadata', 'name') if item_created_at < created_before end rescue ::JSON::ParserError => ex - puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output + puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" [] end def run_command(command) - final_command = ['kubectl', *command.compact].join(' ') - puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output + puts "Running command: `#{command}`" - result = Gitlab::Popen.popen_with_detail([final_command]) + result = Gitlab::Popen.popen_with_detail([command]) if result.status.success? result.stdout.chomp.freeze else - raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}" + raise CommandFailedError, "The `#{command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}" end end end diff --git a/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb b/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb new file mode 100644 index 00000000000..80aa99efc96 --- /dev/null +++ b/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +require 'active_support/inflector' + +require_relative '../helpers/predictive_tests_helper' +require_relative '../../../../lib/gitlab_edition' + +# If a GraphQL type class changed, we try to identify the other GraphQL types that potentially include this type. +module Tooling + module Mappings + class GraphqlBaseTypeMappings + include Helpers::PredictiveTestsHelper + + # Checks for the implements keyword, and graphql_base_types the class name + GRAPHQL_IMPLEMENTS_REGEXP = /implements[( ]([\w:]+)[)]?$/ + + # GraphQL types are a bit scattered in the codebase based on the edition. + # + # Also, a higher edition is able to include lower editions. + # e.g. EE can include FOSS GraphQL types, and JH can include all GraphQL types + GRAPHQL_TYPES_FOLDERS_FOSS = ['app/graphql/types'].freeze + GRAPHQL_TYPES_FOLDERS_EE = GRAPHQL_TYPES_FOLDERS_FOSS + ['ee/app/graphql/types', 'ee/app/graphql/ee/types'] + GRAPHQL_TYPES_FOLDERS_JH = GRAPHQL_TYPES_FOLDERS_EE + ['jh/app/graphql/types', 'jh/app/graphql/jh/types'] + GRAPHQL_TYPES_FOLDERS = { + nil => GRAPHQL_TYPES_FOLDERS_FOSS, + 'ee' => GRAPHQL_TYPES_FOLDERS_EE, + 'jh' => GRAPHQL_TYPES_FOLDERS_JH + }.freeze + + def initialize(changed_files_pathname, predictive_tests_pathname) + @predictive_tests_pathname = predictive_tests_pathname + @changed_files = read_array_from_file(changed_files_pathname) + end + + def execute + # We go through the available editions when searching for base types + # + # `nil` is the FOSS edition + matching_graphql_tests = ([nil] + ::GitlabEdition.extensions).flat_map do |edition| + hierarchy = types_hierarchies[edition] + + filter_files.flat_map do |graphql_file| + children_types = hierarchy[filename_to_class_name(graphql_file)] + next if children_types.empty? + + # We find the specs for the children GraphQL types that are implementing the current GraphQL Type + children_types.map { |filename| filename_to_spec_filename(filename) } + end + end.compact.uniq + + write_array_to_file(predictive_tests_pathname, matching_graphql_tests) + end + + def filter_files + changed_files.select do |filename| + filename.start_with?(*GRAPHQL_TYPES_FOLDERS.values.flatten.uniq) && + filename.end_with?('.rb') && + File.exist?(filename) + end + end + + # Regroup all GraphQL types (by edition) that are implementing another GraphQL type. + # + # The key is the type that is being implemented (e.g. NoteableInterface, TodoableInterface below) + # The value is an array of GraphQL type files that are implementing those types. + # + # Example output: + # + # { + # nil => { + # "NoteableInterface" => [ + # "app/graphql/types/alert_management/alert_type.rb", + # "app/graphql/types/design_management/design_type.rb" + # , "TodoableInterface" => [...] + # }, + # "ee" => { + # "NoteableInterface" => [ + # "app/graphql/types/alert_management/alert_type.rb", + # "app/graphql/types/design_management/design_type.rb", + # "ee/app/graphql/types/epic_type.rb"], + # "TodoableInterface"=> [...] + # } + # } + def types_hierarchies + return @types_hierarchies if @types_hierarchies + + @types_hierarchies = {} + GRAPHQL_TYPES_FOLDERS.each_key do |edition| + @types_hierarchies[edition] = Hash.new { |h, k| h[k] = [] } + + graphql_files_for_edition_glob = File.join("{#{GRAPHQL_TYPES_FOLDERS[edition].join(',')}}", '**', '*.rb') + Dir[graphql_files_for_edition_glob].each do |graphql_file| + graphql_base_types = File.read(graphql_file).scan(GRAPHQL_IMPLEMENTS_REGEXP) + next if graphql_base_types.empty? + + graphql_base_classes = graphql_base_types.flatten.map { |class_name| class_name.split('::').last } + graphql_base_classes.each do |graphql_base_class| + @types_hierarchies[edition][graphql_base_class] += [graphql_file] + end + end + end + + @types_hierarchies + end + + def filename_to_class_name(filename) + File.basename(filename, '.*').camelize + end + + def filename_to_spec_filename(filename) + spec_file = filename.sub('app', 'spec').sub('.rb', '_spec.rb') + + return spec_file if File.exist?(spec_file) + end + + private + + attr_reader :changed_files, :predictive_tests_pathname + end + end +end diff --git a/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb index 365e466011b..bc2cd259fdc 100644 --- a/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb +++ b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb @@ -2,40 +2,49 @@ require 'active_support/inflector' -require_relative 'base' +require_relative '../helpers/predictive_tests_helper' require_relative '../../../../lib/gitlab_edition' # Returns system specs files that are related to the JS files that were changed in the MR. module Tooling module Mappings - class JsToSystemSpecsMappings < Base - def initialize(js_base_folder: 'app/assets/javascripts', system_specs_base_folder: 'spec/features') - @js_base_folder = js_base_folder - @js_base_folders = folders_for_available_editions(js_base_folder) - @system_specs_base_folder = system_specs_base_folder + class JsToSystemSpecsMappings + include Helpers::PredictiveTestsHelper + + def initialize( + changed_files_pathname, predictive_tests_pathname, + js_base_folder: 'app/assets/javascripts', system_specs_base_folder: 'spec/features') + @changed_files = read_array_from_file(changed_files_pathname) + @predictive_tests_pathname = predictive_tests_pathname + @js_base_folder = js_base_folder + @js_base_folders = folders_for_available_editions(js_base_folder) + @system_specs_base_folder = system_specs_base_folder # Cannot be extracted to a constant, as it depends on a variable @first_js_folder_extract_regexp = %r{ (?:.*/)? # Skips the GitLab edition (e.g. ee/, jh/) #{@js_base_folder}/ # Most likely app/assets/javascripts/ + (?:pages/)? # If under a pages folder, we capture the following folder ([\w-]*) # Captures the first folder }x end - def execute(changed_files) - filter_files(changed_files).flat_map do |edition, js_files| + def execute + matching_system_tests = filter_files.flat_map do |edition, js_files| js_keywords_regexp = Regexp.union(construct_js_keywords(js_files)) system_specs_for_edition(edition).select do |system_spec_file| system_spec_file if js_keywords_regexp.match?(system_spec_file) end end + + write_array_to_file(predictive_tests_pathname, matching_system_tests) end # Keep the files that are in the @js_base_folders folders # # Returns a hash, where the key is the GitLab edition, and the values the JS specs - def filter_files(changed_files) + def filter_files selected_files = changed_files.select do |filename| filename.start_with?(*@js_base_folders) && File.exist?(filename) end @@ -54,8 +63,12 @@ module Tooling def system_specs_for_edition(edition) all_files_in_folders_glob = File.join(@system_specs_base_folder, '**', '*') all_files_in_folders_glob = File.join(edition, all_files_in_folders_glob) if edition - Dir[all_files_in_folders_glob].select { |f| File.file?(f) } + Dir[all_files_in_folders_glob].select { |f| File.file?(f) && f.end_with?('_spec.rb') } end + + private + + attr_reader :changed_files, :predictive_tests_pathname end end end diff --git a/tooling/lib/tooling/mappings/partial_to_views_mappings.rb b/tooling/lib/tooling/mappings/partial_to_views_mappings.rb new file mode 100644 index 00000000000..931cacea77f --- /dev/null +++ b/tooling/lib/tooling/mappings/partial_to_views_mappings.rb @@ -0,0 +1,107 @@ +# frozen_string_literal: true + +require_relative '../helpers/predictive_tests_helper' +require_relative '../../../../lib/gitlab_edition' + +# Returns view files that include the potential rails partials from the changed files passed as input. +module Tooling + module Mappings + class PartialToViewsMappings + include Helpers::PredictiveTestsHelper + + def initialize(changed_files_pathname, views_with_partials_pathname, view_base_folder: 'app/views') + @views_with_partials_pathname = views_with_partials_pathname + @changed_files = read_array_from_file(changed_files_pathname) + @view_base_folders = folders_for_available_editions(view_base_folder) + end + + def execute + views_including_modified_partials = [] + + views_globs = view_base_folders.map { |view_base_folder| "#{view_base_folder}/**/*.html.haml" } + Dir[*views_globs].each do |view_file| + included_partial_names = find_pattern_in_file(view_file, partials_keywords_regexp) + next if included_partial_names.empty? + + included_partial_names.each do |included_partial_name| + if view_includes_modified_partial?(view_file, included_partial_name) + views_including_modified_partials << view_file + end + end + end + + write_array_to_file(views_with_partials_pathname, views_including_modified_partials) + end + + def filter_files + @_filter_files ||= changed_files.select do |filename| + filename.start_with?(*view_base_folders) && + File.basename(filename).start_with?('_') && + File.basename(filename).end_with?('.html.haml') && + File.exist?(filename) + end + end + + def partials_keywords_regexp + partial_keywords = filter_files.map do |partial_filename| + extract_partial_keyword(partial_filename) + end + + partial_regexps = partial_keywords.map do |keyword| + %r{(?:render|render_if_exists)(?: |\()(?:partial: ?)?['"]([\w\-_/]*#{keyword})['"]} + end + + Regexp.union(partial_regexps) + end + + # e.g. if app/views/clusters/clusters/_sidebar.html.haml was modified, the partial keyword is `sidebar`. + def extract_partial_keyword(partial_filename) + File.basename(partial_filename).delete_prefix('_').delete_suffix('.html.haml') + end + + # Why do we need this method? + # + # Assume app/views/clusters/clusters/_sidebar.html.haml was modified in the MR. + # + # Suppose now you find = render 'sidebar' in a view. Is this view including the sidebar partial + # that was modified, or another partial called "_sidebar.html.haml" somewhere else? + def view_includes_modified_partial?(view_file, included_partial_name) + view_file_parent_folder = File.dirname(view_file) + included_partial_filename = reconstruct_partial_filename(included_partial_name) + included_partial_relative_path = File.join(view_file_parent_folder, included_partial_filename) + + # We do this because in render (or render_if_exists) + # apparently looks for partials in other GitLab editions + # + # Example: + # + # ee/app/views/events/_epics_filter.html.haml is used in app/views/shared/_event_filter.html.haml + # with render_if_exists 'events/epics_filter' + included_partial_absolute_paths = view_base_folders.map do |view_base_folder| + File.join(view_base_folder, included_partial_filename) + end + + filter_files.include?(included_partial_relative_path) || + (filter_files & included_partial_absolute_paths).any? + end + + def reconstruct_partial_filename(partial_name) + partial_path = partial_name.split('/')[..-2] + partial_filename = partial_name.split('/').last + full_partial_filename = "_#{partial_filename}.html.haml" + + return full_partial_filename if partial_path.empty? + + File.join(partial_path.join('/'), full_partial_filename) + end + + def find_pattern_in_file(file, pattern) + File.read(file).scan(pattern).flatten.compact.uniq + end + + private + + attr_reader :changed_files, :views_with_partials_pathname, :view_base_folders + end + end +end diff --git a/tooling/lib/tooling/mappings/view_to_js_mappings.rb b/tooling/lib/tooling/mappings/view_to_js_mappings.rb index db80eb9bfe8..b78c354f9d2 100644 --- a/tooling/lib/tooling/mappings/view_to_js_mappings.rb +++ b/tooling/lib/tooling/mappings/view_to_js_mappings.rb @@ -1,47 +1,53 @@ # frozen_string_literal: true -require_relative 'base' +require_relative '../helpers/predictive_tests_helper' require_relative '../../../../lib/gitlab_edition' # Returns JS files that are related to the Rails views files that were changed in the MR. module Tooling module Mappings - class ViewToJsMappings < Base + class ViewToJsMappings + include Helpers::PredictiveTestsHelper + # The HTML attribute value pattern we're looking for to match an HTML file to a JS file. HTML_ATTRIBUTE_VALUE_REGEXP = /js-[-\w]+/.freeze # Search for Rails partials included in an HTML file - RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exist)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze + RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exists)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze - def initialize(view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts') - @view_base_folders = folders_for_available_editions(view_base_folder) - @js_base_folders = folders_for_available_editions(js_base_folder) + def initialize( + changed_files_pathname, predictive_tests_pathname, + view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts') + @changed_files = read_array_from_file(changed_files_pathname) + @predictive_tests_pathname = predictive_tests_pathname + @view_base_folders = folders_for_available_editions(view_base_folder) + @js_base_folders = folders_for_available_editions(js_base_folder) end - def execute(changed_files) - changed_view_files = filter_files(changed_files) - - partials = changed_view_files.flat_map do |file| + def execute + partials = filter_files.flat_map do |file| find_partials(file) end - files_to_scan = changed_view_files + partials + files_to_scan = filter_files + partials js_tags = files_to_scan.flat_map do |file| find_pattern_in_file(file, HTML_ATTRIBUTE_VALUE_REGEXP) end js_tags_regexp = Regexp.union(js_tags) - @js_base_folders.flat_map do |js_base_folder| + matching_js_files = @js_base_folders.flat_map do |js_base_folder| Dir["#{js_base_folder}/**/*.{js,vue}"].select do |js_file| file_content = File.read(js_file) js_tags_regexp.match?(file_content) end end + + write_array_to_file(predictive_tests_pathname, matching_js_files) end # Keep the files that are in the @view_base_folders folder - def filter_files(changed_files) - changed_files.select do |filename| + def filter_files + @_filter_files ||= changed_files.select do |filename| filename.start_with?(*@view_base_folders) && File.exist?(filename) end @@ -69,6 +75,10 @@ module Tooling def find_pattern_in_file(file, pattern) File.read(file).scan(pattern).flatten.uniq end + + private + + attr_reader :changed_files, :predictive_tests_pathname end end end diff --git a/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb b/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb new file mode 100644 index 00000000000..1542c817745 --- /dev/null +++ b/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require_relative '../helpers/predictive_tests_helper' +require_relative '../../../../lib/gitlab_edition' + +# Returns system specs files that are related to the Rails views files that were changed in the MR. +module Tooling + module Mappings + class ViewToSystemSpecsMappings + include Helpers::PredictiveTestsHelper + + def initialize(changed_files_pathname, predictive_tests_pathname, view_base_folder: 'app/views') + @predictive_tests_pathname = predictive_tests_pathname + @changed_files = read_array_from_file(changed_files_pathname) + @view_base_folders = folders_for_available_editions(view_base_folder) + end + + def execute + found_system_specs = [] + + filter_files.each do |modified_view_file| + system_specs_exact_match = find_system_specs_exact_match(modified_view_file) + if system_specs_exact_match + found_system_specs << system_specs_exact_match + next + else + system_specs_parent_folder_match = find_system_specs_parent_folder_match(modified_view_file) + found_system_specs += system_specs_parent_folder_match unless system_specs_parent_folder_match.empty? + end + end + + write_array_to_file(predictive_tests_pathname, found_system_specs.compact.uniq.sort) + end + + private + + attr_reader :changed_files, :predictive_tests_pathname, :view_base_folders + + # Keep the views files that are in the @view_base_folders folder + def filter_files + @_filter_files ||= changed_files.select do |filename| + filename.start_with?(*view_base_folders) && + File.basename(filename).end_with?('.html.haml') && + File.exist?(filename) + end + end + + def find_system_specs_exact_match(view_file) + potential_spec_file = to_feature_spec_folder(view_file).sub('.html.haml', '_spec.rb') + + potential_spec_file if File.exist?(potential_spec_file) + end + + def find_system_specs_parent_folder_match(view_file) + parent_system_specs_folder = File.dirname(to_feature_spec_folder(view_file)) + + Dir["#{parent_system_specs_folder}/**/*_spec.rb"] + end + + # e.g. go from app/views/groups/merge_requests.html.haml to spec/features/groups/merge_requests.html.haml + def to_feature_spec_folder(view_file) + view_file.sub(%r{(ee/|jh/)?app/views}, '\1spec/features') + end + end + end +end diff --git a/tooling/lib/tooling/parallel_rspec_runner.rb b/tooling/lib/tooling/parallel_rspec_runner.rb index b1ddc91e831..834d9ec23a7 100644 --- a/tooling/lib/tooling/parallel_rspec_runner.rb +++ b/tooling/lib/tooling/parallel_rspec_runner.rb @@ -1,6 +1,60 @@ # frozen_string_literal: true require 'knapsack' +require 'fileutils' + +module Knapsack + module Distributors + class BaseDistributor + # Refine https://github.com/KnapsackPro/knapsack/blob/v1.21.1/lib/knapsack/distributors/base_distributor.rb + # to take in account the additional filtering we do for predictive jobs. + module BaseDistributorWithTestFiltering + attr_reader :filter_tests + + def initialize(args = {}) + super + + @filter_tests = args[:filter_tests] + end + + def all_tests + @all_tests ||= begin + pattern_tests = Dir.glob(test_file_pattern).uniq + + if filter_tests.empty? + pattern_tests + else + pattern_tests & filter_tests + end + end.sort + end + end + + prepend BaseDistributorWithTestFiltering + end + end + + class AllocatorBuilder + # Refine https://github.com/KnapsackPro/knapsack/blob/v1.21.1/lib/knapsack/allocator_builder.rb + # to take in account the additional filtering we do for predictive jobs. + module AllocatorBuilderWithTestFiltering + attr_accessor :filter_tests + + def allocator + Knapsack::Allocator.new({ + report: Knapsack.report.open, + test_file_pattern: test_file_pattern, + ci_node_total: Knapsack::Config::Env.ci_node_total, + ci_node_index: Knapsack::Config::Env.ci_node_index, + # Additional argument + filter_tests: filter_tests + }) + end + end + + prepend AllocatorBuilderWithTestFiltering + end +end # A custom parallel rspec runner based on Knapsack runner # which takes in additional option for a file containing @@ -13,32 +67,26 @@ require 'knapsack' # would be executed in the CI node. # # Reference: -# https://github.com/ArturT/knapsack/blob/v1.20.0/lib/knapsack/runners/rspec_runner.rb +# https://github.com/ArturT/knapsack/blob/v1.21.1/lib/knapsack/runners/rspec_runner.rb module Tooling class ParallelRSpecRunner def self.run(rspec_args: nil, filter_tests_file: nil) new(rspec_args: rspec_args, filter_tests_file: filter_tests_file).run end - def initialize(allocator: knapsack_allocator, filter_tests_file: nil, rspec_args: nil) - @allocator = allocator + def initialize(filter_tests_file: nil, rspec_args: nil) @filter_tests_file = filter_tests_file @rspec_args = rspec_args&.split(' ') || [] end def run - Knapsack.logger.info - Knapsack.logger.info 'Knapsack node specs:' - Knapsack.logger.info node_tests - Knapsack.logger.info - Knapsack.logger.info 'Filter specs:' - Knapsack.logger.info filter_tests - Knapsack.logger.info - Knapsack.logger.info 'Running specs:' - Knapsack.logger.info tests_to_run - Knapsack.logger.info - - if tests_to_run.empty? + if ENV['KNAPSACK_RSPEC_SUITE_REPORT_PATH'] + knapsack_dir = File.dirname(ENV['KNAPSACK_RSPEC_SUITE_REPORT_PATH']) + FileUtils.mkdir_p(knapsack_dir) + File.write(File.join(knapsack_dir, 'node_specs.txt'), node_tests.join("\n")) + end + + if node_tests.empty? Knapsack.logger.info 'No tests to run on this node, exiting.' return end @@ -50,26 +98,16 @@ module Tooling private - attr_reader :allocator, :filter_tests_file, :rspec_args + attr_reader :filter_tests_file, :rspec_args def rspec_command %w[bundle exec rspec].tap do |cmd| cmd.push(*rspec_args) - cmd.push('--default-path', allocator.test_dir) cmd.push('--') - cmd.push(*tests_to_run) + cmd.push(*node_tests) end end - def tests_to_run - if filter_tests.empty? - Knapsack.logger.info 'Running all node tests without filter' - return node_tests - end - - @tests_to_run ||= node_tests & filter_tests - end - def node_tests allocator.node_tests end @@ -85,8 +123,11 @@ module Tooling File.read(filter_tests_file).split(' ') end - def knapsack_allocator - Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator + def allocator + @allocator ||= + Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).tap do |builder| + builder.filter_tests = filter_tests + end.allocator end end end diff --git a/tooling/lib/tooling/predictive_tests.rb b/tooling/lib/tooling/predictive_tests.rb new file mode 100644 index 00000000000..1ad63e111e3 --- /dev/null +++ b/tooling/lib/tooling/predictive_tests.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require_relative 'find_changes' +require_relative 'find_tests' +require_relative 'find_files_using_feature_flags' +require_relative 'mappings/graphql_base_type_mappings' +require_relative 'mappings/js_to_system_specs_mappings' +require_relative 'mappings/partial_to_views_mappings' +require_relative 'mappings/view_to_js_mappings' +require_relative 'mappings/view_to_system_specs_mappings' + +module Tooling + class PredictiveTests + REQUIRED_ENV_VARIABLES = %w[ + RSPEC_CHANGED_FILES_PATH + RSPEC_MATCHING_TESTS_PATH + RSPEC_VIEWS_INCLUDING_PARTIALS_PATH + FRONTEND_FIXTURES_MAPPING_PATH + RSPEC_MATCHING_JS_FILES_PATH + ].freeze + + def initialize + missing_env_variables = REQUIRED_ENV_VARIABLES.select { |key| ENV[key.to_s] == '' } + unless missing_env_variables.empty? + raise "[predictive tests] Missing ENV variable(s): #{missing_env_variables.join(',')}." + end + + @rspec_changed_files_path = ENV['RSPEC_CHANGED_FILES_PATH'] + @rspec_matching_tests_path = ENV['RSPEC_MATCHING_TESTS_PATH'] + @rspec_views_including_partials_path = ENV['RSPEC_VIEWS_INCLUDING_PARTIALS_PATH'] + @frontend_fixtures_mapping_path = ENV['FRONTEND_FIXTURES_MAPPING_PATH'] + @rspec_matching_js_files_path = ENV['RSPEC_MATCHING_JS_FILES_PATH'] + end + + def execute + Tooling::FindChanges.new( + from: :api, + changed_files_pathname: rspec_changed_files_path + ).execute + Tooling::FindFilesUsingFeatureFlags.new(changed_files_pathname: rspec_changed_files_path).execute + Tooling::FindTests.new(rspec_changed_files_path, rspec_matching_tests_path).execute + Tooling::Mappings::PartialToViewsMappings.new( + rspec_changed_files_path, rspec_views_including_partials_path).execute + Tooling::FindTests.new(rspec_views_including_partials_path, rspec_matching_tests_path).execute + Tooling::Mappings::JsToSystemSpecsMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute + Tooling::Mappings::GraphqlBaseTypeMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute + Tooling::Mappings::ViewToSystemSpecsMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute + Tooling::FindChanges.new( + from: :changed_files, + changed_files_pathname: rspec_changed_files_path, + predictive_tests_pathname: rspec_matching_tests_path, + frontend_fixtures_mapping_pathname: frontend_fixtures_mapping_path + ).execute + Tooling::Mappings::ViewToJsMappings.new(rspec_changed_files_path, rspec_matching_js_files_path).execute + end + + private + + attr_reader :rspec_changed_files_path, :rspec_matching_tests_path, :rspec_views_including_partials_path, + :frontend_fixtures_mapping_path, :rspec_matching_js_files_path + end +end diff --git a/tooling/lib/tooling/test_map_generator.rb b/tooling/lib/tooling/test_map_generator.rb index f96f33ff074..88b4353b232 100644 --- a/tooling/lib/tooling/test_map_generator.rb +++ b/tooling/lib/tooling/test_map_generator.rb @@ -12,7 +12,9 @@ module Tooling def parse(yaml_files) Array(yaml_files).each do |yaml_file| data = File.read(yaml_file) - metadata, example_groups = data.split("---\n").reject(&:empty?).map { |yml| YAML.safe_load(yml, [Symbol]) } + metadata, example_groups = data.split("---\n").reject(&:empty?).map do |yml| + YAML.safe_load(yml, permitted_classes: [Symbol]) + end if example_groups.nil? puts "No examples in #{yaml_file}! Metadata: #{metadata}" diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb index eeda135f3ee..20e00763f65 100644 --- a/tooling/quality/test_level.rb +++ b/tooling/quality/test_level.rb @@ -18,6 +18,7 @@ module Quality unit: %w[ bin channels + components config contracts db @@ -54,7 +55,6 @@ module Quality views workers tooling - components ], integration: %w[ commands @@ -77,8 +77,8 @@ module Quality @patterns[level] ||= "#{prefixes_for_pattern}spec/#{folders_pattern(level)}{,/**/}*#{suffix(level)}".freeze # rubocop:disable Style/RedundantFreeze end - def regexp(level) - @regexps[level] ||= Regexp.new("#{prefixes_for_regex}spec/#{folders_regex(level)}").freeze + def regexp(level, start_with = false) + @regexps[level] ||= Regexp.new("#{'^' if start_with}#{prefixes_for_regex}spec/#{folders_regex(level)}").freeze end def level_for(file_path) diff --git a/tooling/rspec_flaky/config.rb b/tooling/rspec_flaky/config.rb index 36e35671587..0e36e985aad 100644 --- a/tooling/rspec_flaky/config.rb +++ b/tooling/rspec_flaky/config.rb @@ -18,10 +18,6 @@ module RspecFlaky ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec/flaky/new-report.json") end - def self.skipped_flaky_tests_report_path - ENV['SKIPPED_FLAKY_TESTS_REPORT_PATH'] || rails_path("rspec/flaky/skipped_flaky_tests_report.txt") - end - def self.rails_path(path) return path unless defined?(Rails) |