diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-04-20 14:43:17 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-04-20 14:43:17 +0300 |
commit | dfc94207fec2d84314b1a5410cface22e8b369bd (patch) | |
tree | c54022f61ced104305889a64de080998a0dc773b /tooling | |
parent | b874efeff674f6bf0355d5d242ecf81c6f7155df (diff) |
Add latest changes from gitlab-org/gitlab@15-11-stable-eev15.11.0-rc42
Diffstat (limited to 'tooling')
29 files changed, 919 insertions, 370 deletions
diff --git a/tooling/bin/find_changes b/tooling/bin/find_changes deleted file mode 100755 index 38e1f363dd9..00000000000 --- a/tooling/bin/find_changes +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'gitlab' - -class FindChanges # rubocop:disable Gitlab/NamespacedClass - def initialize(output_file:, matched_tests_file: nil, frontend_fixtures_mapping_path: nil) - @gitlab_token = ENV.fetch('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE', '') - @gitlab_endpoint = ENV.fetch('CI_API_V4_URL') - @mr_project_path = ENV.fetch('CI_MERGE_REQUEST_PROJECT_PATH') - @mr_iid = ENV.fetch('CI_MERGE_REQUEST_IID') - @output_file = output_file - @matched_tests_file = matched_tests_file - @frontend_fixtures_mapping_path = frontend_fixtures_mapping_path - end - - def execute - add_frontend_fixture_files! - - File.write(output_file, file_changes.join(' ')) - end - - private - - def add_frontend_fixture_files? - matched_tests_file && frontend_fixtures_mapping_path - end - - def add_frontend_fixture_files! - return unless add_frontend_fixture_files? - - # If we have a `test file -> JSON frontend fixture` mapping file, we add the files JSON frontend fixtures - # files to the list of changed files so that Jest can automatically run the dependent tests thanks to --findRelatedTests - test_files.each do |test_file| - file_changes.concat(frontend_fixtures_mapping[test_file]) if frontend_fixtures_mapping.key?(test_file) - end - end - - def file_changes - @file_changes ||= - if File.exist?(output_file) - File.read(output_file).split(' ') - else - Gitlab.configure do |config| - config.endpoint = gitlab_endpoint - config.private_token = gitlab_token - end - - mr_changes.changes.flat_map do |change| - change.to_h.values_at('old_path', 'new_path') - end.uniq - end - end - - def mr_changes - @mr_changes ||= Gitlab.merge_request_changes(mr_project_path, mr_iid) - end - - def test_files - return [] if !matched_tests_file || !File.exist?(matched_tests_file) - - File.read(matched_tests_file).split(' ') - end - - def frontend_fixtures_mapping - return {} if !frontend_fixtures_mapping_path || !File.exist?(frontend_fixtures_mapping_path) - - JSON.parse(File.read(frontend_fixtures_mapping_path)) # rubocop:disable Gitlab/Json - end - - attr_reader :gitlab_token, :gitlab_endpoint, :mr_project_path, :mr_iid, :output_file, :matched_tests_file, :frontend_fixtures_mapping_path -end - -output_file = ARGV.shift -raise ArgumentError, "An path to an output file must be given as first argument of #{__FILE__}." if output_file.nil? - -matched_tests_file = ARGV.shift -frontend_fixtures_mapping_path = ARGV.shift - -FindChanges - .new(output_file: output_file, matched_tests_file: matched_tests_file, frontend_fixtures_mapping_path: frontend_fixtures_mapping_path) - .execute diff --git a/tooling/bin/find_only_js_changes b/tooling/bin/find_only_js_changes new file mode 100755 index 00000000000..a69ee64fe14 --- /dev/null +++ b/tooling/bin/find_only_js_changes @@ -0,0 +1,12 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative '../lib/tooling/find_changes' + +if Tooling::FindChanges.new(from: :api).only_js_files_changed + puts "Only JS files were changed" + exit 0 +else + puts "Changes were made to files other than JS files" + exit 1 +end diff --git a/tooling/bin/find_tests b/tooling/bin/find_tests deleted file mode 100755 index 33834064f36..00000000000 --- a/tooling/bin/find_tests +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'test_file_finder' - -changes = ARGV.shift -output_file = ARGV.shift - -changed_files = File.read(changes).split(' ') - -tff = TestFileFinder::FileFinder.new(paths: changed_files).tap do |file_finder| - file_finder.use TestFileFinder::MappingStrategies::PatternMatching.load('tests.yml') - - if ENV['RSPEC_TESTS_MAPPING_ENABLED'] - file_finder.use TestFileFinder::MappingStrategies::DirectMatching.load_json(ENV['RSPEC_TESTS_MAPPING_PATH']) - end -end - -File.write(output_file, tff.test_files.uniq.join(' ')) diff --git a/tooling/bin/gettext_extractor b/tooling/bin/gettext_extractor new file mode 100755 index 00000000000..39f029616df --- /dev/null +++ b/tooling/bin/gettext_extractor @@ -0,0 +1,29 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative '../lib/tooling/gettext_extractor' + +pot_file = ARGV.shift + +if !pot_file || !Dir.exist?(File.dirname(pot_file)) + abort <<~MSG + Please provide a target file name as the first argument, e.g. + #{$PROGRAM_NAME} locale/gitlab.pot + MSG +end + +puts <<~MSG + Extracting translatable strings from source files... +MSG + +root_dir = File.expand_path('../../', __dir__) + +extractor = Tooling::GettextExtractor.new( + glob_base: root_dir +) + +File.write(pot_file, extractor.generate_pot) + +puts <<~MSG + All done. Please commit the changes to `#{pot_file}`. +MSG diff --git a/tooling/bin/js_to_system_specs_mappings b/tooling/bin/js_to_system_specs_mappings deleted file mode 100755 index 3e9d9cb4c5f..00000000000 --- a/tooling/bin/js_to_system_specs_mappings +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require_relative '../lib/tooling/mappings/js_to_system_specs_mappings' - -changes = ARGV.shift -matching_tests = ARGV.shift - -changed_files = File.read(changes).split(' ') -matching_test_files = File.read(matching_tests).split(' ') - -system_tests = Tooling::Mappings::JsToSystemSpecsMappings.new.execute(changed_files) - -File.write(matching_tests, (matching_test_files + system_tests).join(' ')) diff --git a/tooling/bin/predictive_tests b/tooling/bin/predictive_tests new file mode 100755 index 00000000000..61543494a69 --- /dev/null +++ b/tooling/bin/predictive_tests @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative '../lib/tooling/predictive_tests' + +Tooling::PredictiveTests.new.execute diff --git a/tooling/bin/view_to_js_mappings b/tooling/bin/view_to_js_mappings deleted file mode 100755 index 483003aac5e..00000000000 --- a/tooling/bin/view_to_js_mappings +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require_relative '../lib/tooling/mappings/view_to_js_mappings' - -changes = ARGV.shift -output_file = ARGV.shift -changed_files = File.read(changes).split(' ') - -File.write(output_file, Tooling::Mappings::ViewToJsMappings.new.execute(changed_files).join(' ')) diff --git a/tooling/danger/feature_flag.rb b/tooling/danger/feature_flag.rb index da0b7053af1..3fb20c561af 100644 --- a/tooling/danger/feature_flag.rb +++ b/tooling/danger/feature_flag.rb @@ -15,6 +15,12 @@ module Tooling files.select { |path| path =~ %r{\A(ee/)?config/feature_flags/} }.map { |path| Found.new(path) } end + # TODO: Move this to gitlab-dangerfiles helper + # https://gitlab.com/gitlab-org/ruby/gems/gitlab-dangerfiles/-/blob/master/lib/danger/plugins/internal/helper.rb + def stage_label + helper.mr_labels.find { |label| label.start_with?("devops::") } + end + class Found ATTRIBUTES = %w[name introduced_by_url rollout_issue_url milestone type group default_enabled].freeze diff --git a/tooling/danger/multiversion.rb b/tooling/danger/multiversion.rb new file mode 100644 index 00000000000..612881aaa1a --- /dev/null +++ b/tooling/danger/multiversion.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Tooling + module Danger + module Multiversion + FRONTEND_REGEX = %r{\A((ee|jh)/)?app/assets/.*(\.(vue|js|graphql))\z} + GRAPHQL_BACKEND_REGEX = %r{\A((ee|jh)/)?app/graphql/} + + def check! + return unless helper.ci? + return unless frontend_changed? && backend_changed? + + markdown <<~MARKDOWN + ## Multiversion compatibility + + This merge request updates GraphQL backend and frontend code. + + To prevent an incident, ensure the updated frontend code is backwards compatible. + + For more information, see the [multiversion compatibility documentation](https://docs.gitlab.com/ee/development/graphql_guide/reviewing.html#multiversion-compatibility). + MARKDOWN + end + + private + + def frontend_changed? + !git.modified_files.grep(FRONTEND_REGEX).empty? || !git.added_files.grep(FRONTEND_REGEX).empty? + end + + def backend_changed? + !git.added_files.grep(GRAPHQL_BACKEND_REGEX).empty? || !git.modified_files.grep(GRAPHQL_BACKEND_REGEX).empty? + end + end + end +end diff --git a/tooling/danger/specs.rb b/tooling/danger/specs.rb index 5359e71f8cc..68cab141656 100644 --- a/tooling/danger/specs.rb +++ b/tooling/danger/specs.rb @@ -1,55 +1,19 @@ # frozen_string_literal: true -require_relative 'suggestor' +Dir[File.expand_path('specs/*_suggestion.rb', __dir__)].each { |file| require file } module Tooling module Danger module Specs - include ::Tooling::Danger::Suggestor - SPEC_FILES_REGEX = 'spec/' EE_PREFIX = 'ee/' - PROJECT_FACTORIES = %w[ - :project - :project_empty_repo - :forked_project_with_submodules - :project_with_design + SUGGESTIONS = [ + FeatureCategorySuggestion, + MatchWithArraySuggestion, + ProjectFactorySuggestion ].freeze - PROJECT_FACTORY_REGEX = / - ^\+? # Start of the line, which may or may not have a `+` - (?<head>\s*) # 0-many leading whitespace captured in a group named head - let!? # Literal `let` which may or may not end in `!` - (?<tail> # capture group named tail - \([^)]+\) # Two parenthesis with any non-parenthesis characters between them - \s*\{\s* # Opening curly brace surrounded by 0-many whitespace characters - create\( # literal - (?:#{PROJECT_FACTORIES.join('|')}) # Any of the project factory names - \W # Non-word character, avoid matching factories like :project_authorization - ) # end capture group named tail - /x.freeze - - PROJECT_FACTORY_REPLACEMENT = '\k<head>let_it_be\k<tail>' - PROJECT_FACTORY_SUGGESTION = <<~SUGGEST_COMMENT - Project creations are very slow. Use `let_it_be`, `build` or `build_stubbed` if possible. - See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#optimize-factory-usage) - for background information and alternative options. - SUGGEST_COMMENT - - MATCH_WITH_ARRAY_REGEX = /(?<to>to\(?\s*)(?<matcher>match|eq)(?<expectation>[( ]?\[(?=.*,)[^\]]+)/.freeze - MATCH_WITH_ARRAY_REPLACEMENT = '\k<to>match_array\k<expectation>' - MATCH_WITH_ARRAY_SUGGESTION = <<~SUGGEST_COMMENT - If order of the result is not important, please consider using `match_array` to avoid flakiness. - SUGGEST_COMMENT - - RSPEC_TOP_LEVEL_DESCRIBE_REGEX = /^\+.?RSpec\.describe(.+)/.freeze - FEATURE_CATEGORY_SUGGESTION = <<~SUGGESTION_MARKDOWN - Consider adding `feature_category: <feature_category_name>` for this example if it is not set already. - See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#feature-category-metadata). - SUGGESTION_MARKDOWN - FEATURE_CATEGORY_KEYWORD = 'feature_category' - def changed_specs_files(ee: :include) changed_files = helper.all_changed_files folder_prefix = @@ -65,45 +29,9 @@ module Tooling changed_files.grep(%r{\A#{folder_prefix}#{SPEC_FILES_REGEX}}) end - def add_suggestions_for_match_with_array(filename) - add_suggestion( - filename: filename, - regex: MATCH_WITH_ARRAY_REGEX, - replacement: MATCH_WITH_ARRAY_REPLACEMENT, - comment_text: MATCH_WITH_ARRAY_SUGGESTION - ) - end - - def add_suggestions_for_project_factory_usage(filename) - add_suggestion( - filename: filename, - regex: PROJECT_FACTORY_REGEX, - replacement: PROJECT_FACTORY_REPLACEMENT, - comment_text: PROJECT_FACTORY_SUGGESTION - ) - end - - def add_suggestions_for_feature_category(filename) - file_lines = project_helper.file_lines(filename) - changed_lines = helper.changed_lines(filename) - - changed_lines.each_with_index do |changed_line, i| - next unless changed_line =~ RSPEC_TOP_LEVEL_DESCRIBE_REGEX - - line_number = file_lines.find_index(changed_line.delete_prefix('+')) - next unless line_number - - # Get the top level RSpec.describe line and the next 5 lines - lines_to_check = file_lines[line_number, 5] - # Remove all the lines after the first one that ends in `do` - last_line_number_of_describe_declaration = lines_to_check.index { |line| line.end_with?(' do') } - lines_to_check = lines_to_check[0..last_line_number_of_describe_declaration] - - next if lines_to_check.any? { |line| line.include?(FEATURE_CATEGORY_KEYWORD) } - - suggested_line = file_lines[line_number] - - markdown(comment(FEATURE_CATEGORY_SUGGESTION, suggested_line), file: filename, line: line_number.succ) + def add_suggestions_for(filename) + SUGGESTIONS.each do |suggestion| + suggestion.new(filename, context: self).suggest end end end diff --git a/tooling/danger/specs/feature_category_suggestion.rb b/tooling/danger/specs/feature_category_suggestion.rb new file mode 100644 index 00000000000..5acf73c8956 --- /dev/null +++ b/tooling/danger/specs/feature_category_suggestion.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require_relative '../suggestion' + +module Tooling + module Danger + module Specs + class FeatureCategorySuggestion < Suggestion + RSPEC_TOP_LEVEL_DESCRIBE_REGEX = /^\+.?RSpec\.describe(.+)/ + SUGGESTION = <<~SUGGESTION_MARKDOWN + Consider adding `feature_category: <feature_category_name>` for this example if it is not set already. + See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#feature-category-metadata). + SUGGESTION_MARKDOWN + FEATURE_CATEGORY_KEYWORD = 'feature_category' + + def suggest + file_lines = project_helper.file_lines(filename) + changed_lines = helper.changed_lines(filename) + + changed_lines.each do |changed_line| + next unless changed_line =~ RSPEC_TOP_LEVEL_DESCRIBE_REGEX + + line_number = file_lines.find_index(changed_line.delete_prefix('+')) + next unless line_number + + # Get the top level RSpec.describe line and the next 5 lines + lines_to_check = file_lines[line_number, 5] + # Remove all the lines after the first one that ends in `do` + last_line_number_of_describe_declaration = lines_to_check.index { |line| line.end_with?(' do') } + lines_to_check = lines_to_check[0..last_line_number_of_describe_declaration] + + next if lines_to_check.any? { |line| line.include?(FEATURE_CATEGORY_KEYWORD) } + + suggested_line = file_lines[line_number] + + markdown(comment(SUGGESTION, suggested_line), file: filename, line: line_number.succ) + end + end + end + end + end +end diff --git a/tooling/danger/specs/match_with_array_suggestion.rb b/tooling/danger/specs/match_with_array_suggestion.rb new file mode 100644 index 00000000000..eb0f7a1a832 --- /dev/null +++ b/tooling/danger/specs/match_with_array_suggestion.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require_relative '../suggestion' + +module Tooling + module Danger + module Specs + class MatchWithArraySuggestion < Suggestion + MATCH = /(?<to>to\(?\s*)(?<matcher>match|eq)(?<expectation>[( ]?\[(?=.*,)[^\]]+)/ + REPLACEMENT = '\k<to>match_array\k<expectation>' + SUGGESTION = <<~SUGGEST_COMMENT + If order of the result is not important, please consider using `match_array` to avoid flakiness. + SUGGEST_COMMENT + end + end + end +end diff --git a/tooling/danger/specs/project_factory_suggestion.rb b/tooling/danger/specs/project_factory_suggestion.rb new file mode 100644 index 00000000000..4e5a70ac8e5 --- /dev/null +++ b/tooling/danger/specs/project_factory_suggestion.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require_relative '../suggestion' + +module Tooling + module Danger + module Specs + class ProjectFactorySuggestion < Suggestion + PROJECT_FACTORIES = %w[ + :project + :project_empty_repo + :forked_project_with_submodules + :project_with_design + ].freeze + + MATCH = / + ^\+? # Start of the line, which may or may not have a `+` + (?<head>\s*) # 0-many leading whitespace captured in a group named head + let!? # Literal `let` which may or may not end in `!` + (?<tail> # capture group named tail + \([^)]+\) # Two parenthesis with any non-parenthesis characters between them + \s*\{\s* # Opening curly brace surrounded by 0-many whitespace characters + create\( # literal + (?:#{PROJECT_FACTORIES.join('|')}) # Any of the project factory names + \W # Non-word character, avoid matching factories like :project_badge + ) # end capture group named tail + /x + + REPLACEMENT = '\k<head>let_it_be\k<tail>' + SUGGESTION = <<~SUGGEST_COMMENT + Project creations are very slow. Use `let_it_be`, `build` or `build_stubbed` if possible. + See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#optimize-factory-usage) + for background information and alternative options. + SUGGEST_COMMENT + end + end + end +end diff --git a/tooling/danger/stable_branch.rb b/tooling/danger/stable_branch.rb index 2751a6f0191..9deb4838079 100644 --- a/tooling/danger/stable_branch.rb +++ b/tooling/danger/stable_branch.rb @@ -85,12 +85,12 @@ module Tooling !has_flaky_failure_label? end - private - def valid_stable_branch? !!stable_target_branch && !helper.security_mr? end + private + def package_and_test_bridge_and_pipeline_status mr_head_pipeline_id = gitlab.mr_json.dig('head_pipeline', 'id') return unless mr_head_pipeline_id diff --git a/tooling/danger/suggestion.rb b/tooling/danger/suggestion.rb new file mode 100644 index 00000000000..da3c6b0e76f --- /dev/null +++ b/tooling/danger/suggestion.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +require 'forwardable' +require_relative 'suggestor' + +module Tooling + module Danger + # A basic suggestion. + # + # A subclass needs to define the following constants: + # * MATCH (Regexp) - A Regexp to match file lines + # * REPLACEMENT (String) - A suggestion replacement text + # * SUGGESTION (String) - A suggestion text + # + # @see Suggestor + class Suggestion + extend Forwardable + include ::Tooling::Danger::Suggestor + + def_delegators :@context, :helper, :project_helper, :markdown + + attr_reader :filename + + def initialize(filename, context:) + @filename = filename + @context = context + end + + def suggest + add_suggestion( + filename: filename, + regex: self.class::MATCH, + replacement: self.class::REPLACEMENT, + comment_text: self.class::SUGGESTION + ) + end + end + end +end diff --git a/tooling/docs/deprecation_handling.rb b/tooling/docs/deprecation_handling.rb index bcdf73e0044..5996a0c89c1 100644 --- a/tooling/docs/deprecation_handling.rb +++ b/tooling/docs/deprecation_handling.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'erb' module Docs @@ -7,11 +8,7 @@ module Docs @type = type @yaml_glob_path = Rails.root.join("data/#{type.pluralize}/**/*.yml") @template_path = Rails.root.join("data/#{type.pluralize}/templates/_#{type}_template.md.erb") - @milestone_key_name = if type == "deprecation" - "announcement_milestone" - else - "removal_milestone" - end + @milestone_key_name = "removal_milestone" end def render diff --git a/tooling/lib/tooling/find_changes.rb b/tooling/lib/tooling/find_changes.rb new file mode 100755 index 00000000000..d8373d11245 --- /dev/null +++ b/tooling/lib/tooling/find_changes.rb @@ -0,0 +1,107 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'gitlab' +require_relative 'helpers/file_handler' + +module Tooling + class FindChanges + include Helpers::FileHandler + + def initialize( + from:, + changed_files_pathname: nil, + predictive_tests_pathname: nil, + frontend_fixtures_mapping_pathname: nil + ) + + raise ArgumentError, ':from can only be :api or :changed_files' unless + %i[api changed_files].include?(from) + + @gitlab_token = ENV['PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE'] || '' + @gitlab_endpoint = ENV['CI_API_V4_URL'] + @mr_project_path = ENV['CI_MERGE_REQUEST_PROJECT_PATH'] + @mr_iid = ENV['CI_MERGE_REQUEST_IID'] + @changed_files_pathname = changed_files_pathname + @predictive_tests_pathname = predictive_tests_pathname + @frontend_fixtures_mapping_pathname = frontend_fixtures_mapping_pathname + @from = from + end + + def execute + if changed_files_pathname.nil? + raise ArgumentError, "A path to the changed files file must be given as :changed_files_pathname" + end + + case @from + when :api + write_array_to_file(changed_files_pathname, file_changes + frontend_fixture_files, append: false) + else + write_array_to_file(changed_files_pathname, frontend_fixture_files, append: true) + end + end + + def only_js_files_changed + file_changes.any? && file_changes.all? { |file| file.end_with?('.js') } + end + + private + + attr_reader :gitlab_token, :gitlab_endpoint, :mr_project_path, + :mr_iid, :changed_files_pathname, :predictive_tests_pathname, :frontend_fixtures_mapping_pathname + + def gitlab + @gitlab ||= begin + Gitlab.configure do |config| + config.endpoint = gitlab_endpoint + config.private_token = gitlab_token + end + + Gitlab + end + end + + def add_frontend_fixture_files? + predictive_tests_pathname && frontend_fixtures_mapping_pathname + end + + def frontend_fixture_files + # If we have a `test file -> JSON frontend fixture` mapping file, we add the files JSON frontend fixtures + # files to the list of changed files so that Jest can automatically run the dependent tests + # using --findRelatedTests flag. + empty = [].freeze + + test_files.flat_map do |test_file| + frontend_fixtures_mapping[test_file] || empty + end + end + + def file_changes + @file_changes ||= + case @from + when :api + mr_changes.changes.flat_map do |change| + change.to_h.values_at('old_path', 'new_path') + end.uniq + else + read_array_from_file(changed_files_pathname) + end + end + + def mr_changes + @mr_changes ||= gitlab.merge_request_changes(mr_project_path, mr_iid) + end + + def test_files + return [] if !predictive_tests_pathname || !File.exist?(predictive_tests_pathname) + + read_array_from_file(predictive_tests_pathname) + end + + def frontend_fixtures_mapping + return {} if !frontend_fixtures_mapping_pathname || !File.exist?(frontend_fixtures_mapping_pathname) + + JSON.parse(File.read(frontend_fixtures_mapping_pathname)) # rubocop:disable Gitlab/Json + end + end +end diff --git a/tooling/lib/tooling/find_tests.rb b/tooling/lib/tooling/find_tests.rb new file mode 100644 index 00000000000..f26c1eacdc7 --- /dev/null +++ b/tooling/lib/tooling/find_tests.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'test_file_finder' +require_relative 'helpers/file_handler' + +module Tooling + class FindTests + include Helpers::FileHandler + + def initialize(changed_files_pathname, predictive_tests_pathname) + @predictive_tests_pathname = predictive_tests_pathname + @changed_files = read_array_from_file(changed_files_pathname) + end + + def execute + tff = TestFileFinder::FileFinder.new(paths: changed_files).tap do |file_finder| + file_finder.use TestFileFinder::MappingStrategies::PatternMatching.load('tests.yml') + + if ENV['RSPEC_TESTS_MAPPING_ENABLED'] == 'true' + file_finder.use TestFileFinder::MappingStrategies::DirectMatching.load_json(ENV['RSPEC_TESTS_MAPPING_PATH']) + end + end + + write_array_to_file(predictive_tests_pathname, tff.test_files.uniq) + end + + private + + attr_reader :changed_files, :matching_tests, :predictive_tests_pathname + end +end diff --git a/tooling/lib/tooling/gettext_extractor.rb b/tooling/lib/tooling/gettext_extractor.rb new file mode 100644 index 00000000000..673749a5a16 --- /dev/null +++ b/tooling/lib/tooling/gettext_extractor.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +require 'parallel' +require 'gettext/po' +require 'gettext/po_entry' +require 'gettext/tools/parser/erb' +require 'gettext/tools/parser/ruby' +require 'gettext_i18n_rails/haml_parser' +require 'json' +require 'open3' + +module Tooling + class GettextExtractor < GetText::Tools::XGetText + class HamlParser < GettextI18nRails::HamlParser + # If both `haml` and `hamlit` are available, + # the parser prefers `haml`. `hamlit` should be faster + def self.libraries + ["hamlit"] + end + end + + def initialize( + backend_glob: "{ee,app,lib,config,locale}/**/*.{rb,erb,haml}", + glob_base: nil, + package_name: 'gitlab', + package_version: '1.0.0' + ) + super() + @backend_glob = backend_glob + @package_name = package_name + @glob_base = glob_base || Dir.pwd + @package_version = package_version + # Ensure that the messages are ordered by id + @po_order = :msgid + @po_format_options = { + # No line breaks within a message + max_line_width: -1, + # Do not print references to files + include_reference_comment: false + } + end + + def parse(_paths) + po = GetText::PO.new + parse_backend_files.each do |po_entry| + merge_po_entries(po, po_entry) + end + parse_frontend_files.each do |po_entry| + merge_po_entries(po, po_entry) + end + po + end + + # Overrides method from GetText::Tools::XGetText + # This makes a method public and passes in an empty array of paths, + # as our overidden "parse" method needs no paths + def generate_pot + super([]) + end + + private + + # Overrides method from GetText::Tools::XGetText + # in order to remove revision dates, as we check in our locale/gitlab.pot + def header_content + super.gsub(/^POT?-(?:Creation|Revision)-Date:.*\n/, '') + end + + def merge_po_entries(po, po_entry) + existing_entry = po[po_entry.msgctxt, po_entry.msgid] + po_entry = existing_entry.merge(po_entry) if existing_entry + + po[po_entry.msgctxt, po_entry.msgid] = po_entry + end + + def parse_backend_file(path) + case ::File.extname(path) + when '.rb' + GetText::RubyParser.new(path).parse + when '.haml' + HamlParser.parse(path).collect { |item| create_po_entry(*item) } + when '.erb' + GetText::ErbParser.new(path).parse + else + raise NotImplementedError + end + end + + def parse_backend_files + files = Dir.glob(File.join(@glob_base, @backend_glob)) + Parallel.flat_map(files) { |item| parse_backend_file(item) } + end + + def parse_frontend_files + results, status = Open3.capture2('node scripts/frontend/extract_gettext_all.js --all') + raise StandardError, "Could not parse frontend files" unless status.success? + + # rubocop:disable Gitlab/Json + JSON.parse(results) + .values + .flatten(1) + .collect { |entry| create_po_entry(*entry) } + # rubocop:enable Gitlab/Json + end + end +end diff --git a/tooling/lib/tooling/helpers/file_handler.rb b/tooling/lib/tooling/helpers/file_handler.rb new file mode 100644 index 00000000000..88248e31df2 --- /dev/null +++ b/tooling/lib/tooling/helpers/file_handler.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'fileutils' + +module Tooling + module Helpers + module FileHandler + def read_array_from_file(file) + FileUtils.touch file + + File.read(file).split(' ') + end + + def write_array_to_file(file, content_array, append: true) + FileUtils.touch file + + # We sort the array to make it easier to read the output file + content_array.sort! + + output_content = + if append + [File.read(file), *content_array].join(' ').lstrip + else + content_array.join(' ') + end + + File.write(file, output_content) + end + end + end +end diff --git a/tooling/lib/tooling/kubernetes_client.rb b/tooling/lib/tooling/kubernetes_client.rb index 27eb4c8151e..5579f130a84 100644 --- a/tooling/lib/tooling/kubernetes_client.rb +++ b/tooling/lib/tooling/kubernetes_client.rb @@ -6,33 +6,23 @@ require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen) module Tooling class KubernetesClient - RESOURCE_LIST = 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd' K8S_ALLOWED_NAMESPACES_REGEX = /^review-(?!apps).+/.freeze CommandFailedError = Class.new(StandardError) - attr_reader :namespace + def cleanup_namespaces_by_created_at(created_before:) + stale_namespaces = namespaces_created_before(created_before: created_before) - def initialize(namespace:) - @namespace = namespace - end - - def cleanup_by_release(release_name:, wait: true) - delete_by_selector(release_name: release_name, wait: wait) - delete_by_matching_name(release_name: release_name) - end - - def cleanup_by_created_at(resource_type:, created_before:, wait: true) - resource_names = resource_names_created_before(resource_type: resource_type, created_before: created_before) - return if resource_names.empty? + # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby. + review_apps_stale_namespaces = stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) } + return if review_apps_stale_namespaces.empty? - delete_by_exact_names(resource_type: resource_type, resource_names: resource_names, wait: wait) + delete_namespaces(review_apps_stale_namespaces) end - def cleanup_review_app_namespaces(created_before:, wait: true) - namespaces = review_app_namespaces_created_before(created_before: created_before) - return if namespaces.empty? + def delete_namespaces(namespaces) + return if namespaces.any? { |ns| !K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) } - delete_namespaces_by_exact_names(resource_names: namespaces, wait: wait) + run_command("kubectl delete namespace --now --ignore-not-found #{namespaces.join(' ')}") end def delete_namespaces_by_exact_names(resource_names:, wait:) @@ -48,126 +38,29 @@ module Tooling run_command(command) end - private - - def delete_by_selector(release_name:, wait:) - selector = case release_name - when String - %(-l release="#{release_name}") - when Array - %(-l 'release in (#{release_name.join(', ')})') - else - raise ArgumentError, 'release_name must be a string or an array' - end - - command = [ - 'delete', - RESOURCE_LIST, - %(--namespace "#{namespace}"), - '--now', - '--ignore-not-found', - %(--wait=#{wait}), - selector - ] - - run_command(command) - end - - def delete_by_exact_names(resource_names:, wait:, resource_type: nil) - command = [ - 'delete', - resource_type, - %(--namespace "#{namespace}"), - '--now', - '--ignore-not-found', - %(--wait=#{wait}), - resource_names.join(' ') - ] - - run_command(command) - end - - def delete_by_matching_name(release_name:) - resource_names = raw_resource_names - command = [ - 'delete', - %(--namespace "#{namespace}"), - '--ignore-not-found' - ] - - Array(release_name).each do |release| - resource_names - .select { |resource_name| resource_name.include?(release) } - .each { |matching_resource| run_command(command + [matching_resource]) } - end - end - - def raw_resource_names - command = [ - 'get', - RESOURCE_LIST, - %(--namespace "#{namespace}"), - '-o name' - ] - run_command(command).lines.map(&:strip) - end - - def resource_names_created_before(resource_type:, created_before:) - command = [ - 'get', - resource_type, - %(--namespace "#{namespace}"), - "--sort-by='{.metadata.creationTimestamp}'", - '-o json' - ] + def namespaces_created_before(created_before:) + response = run_command("kubectl get namespace --all-namespaces --sort-by='{.metadata.creationTimestamp}' -o json") - response = run_command(command) - - resources_created_before_date(response, created_before) - end - - def review_app_namespaces_created_before(created_before:) - command = [ - 'get', - 'namespace', - "--sort-by='{.metadata.creationTimestamp}'", - '-o json' - ] - - response = run_command(command) - - stale_namespaces = resources_created_before_date(response, created_before) - - # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby. - stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) } - end - - def resources_created_before_date(response, date) items = JSON.parse(response)['items'] # rubocop:disable Gitlab/Json - - items.each_with_object([]) do |item, result| + items.filter_map do |item| item_created_at = Time.parse(item.dig('metadata', 'creationTimestamp')) - if item_created_at < date - resource_name = item.dig('metadata', 'name') - result << resource_name - end + item.dig('metadata', 'name') if item_created_at < created_before end rescue ::JSON::ParserError => ex - puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output + puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" [] end def run_command(command) - final_command = ['kubectl', *command.compact].join(' ') - puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output + puts "Running command: `#{command}`" - result = Gitlab::Popen.popen_with_detail([final_command]) + result = Gitlab::Popen.popen_with_detail([command]) if result.status.success? result.stdout.chomp.freeze else - raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}" + raise CommandFailedError, "The `#{command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}" end end end diff --git a/tooling/lib/tooling/mappings/base.rb b/tooling/lib/tooling/mappings/base.rb index 93d3a967114..27a9a0925b0 100644 --- a/tooling/lib/tooling/mappings/base.rb +++ b/tooling/lib/tooling/mappings/base.rb @@ -1,22 +1,13 @@ # frozen_string_literal: true require_relative '../../../../lib/gitlab_edition' +require_relative '../helpers/file_handler' # Returns system specs files that are related to the JS files that were changed in the MR. module Tooling module Mappings class Base - # Input: A list of space-separated files - # Output: A list of space-separated specs files (JS, Ruby, ...) - def execute(changed_files) - raise "Not Implemented" - end - - # Input: A list of space-separated files - # Output: array/hash of files - def filter_files(changed_files) - raise "Not Implemented" - end + include Helpers::FileHandler # Input: A folder # Output: An array of folders, each prefixed with a GitLab edition diff --git a/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb b/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb new file mode 100644 index 00000000000..569a8278163 --- /dev/null +++ b/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +require 'active_support/inflector' + +require_relative 'base' +require_relative '../../../../lib/gitlab_edition' + +# If a GraphQL type class changed, we try to identify the other GraphQL types that potentially include this type. +module Tooling + module Mappings + class GraphqlBaseTypeMappings < Base + # Checks for the implements keyword, and graphql_base_types the class name + GRAPHQL_IMPLEMENTS_REGEXP = /implements[( ]([\w:]+)[)]?$/ + + # GraphQL types are a bit scattered in the codebase based on the edition. + # + # Also, a higher edition is able to include lower editions. + # e.g. EE can include FOSS GraphQL types, and JH can include all GraphQL types + GRAPHQL_TYPES_FOLDERS_FOSS = ['app/graphql/types'].freeze + GRAPHQL_TYPES_FOLDERS_EE = GRAPHQL_TYPES_FOLDERS_FOSS + ['ee/app/graphql/types', 'ee/app/graphql/ee/types'] + GRAPHQL_TYPES_FOLDERS_JH = GRAPHQL_TYPES_FOLDERS_EE + ['jh/app/graphql/types', 'jh/app/graphql/jh/types'] + GRAPHQL_TYPES_FOLDERS = { + nil => GRAPHQL_TYPES_FOLDERS_FOSS, + 'ee' => GRAPHQL_TYPES_FOLDERS_EE, + 'jh' => GRAPHQL_TYPES_FOLDERS_JH + }.freeze + + def initialize(changed_files_pathname, predictive_tests_pathname) + @predictive_tests_pathname = predictive_tests_pathname + @changed_files = read_array_from_file(changed_files_pathname) + end + + def execute + # We go through the available editions when searching for base types + # + # `nil` is the FOSS edition + matching_graphql_tests = ([nil] + ::GitlabEdition.extensions).flat_map do |edition| + hierarchy = types_hierarchies[edition] + + filter_files.flat_map do |graphql_file| + children_types = hierarchy[filename_to_class_name(graphql_file)] + next if children_types.empty? + + # We find the specs for the children GraphQL types that are implementing the current GraphQL Type + children_types.map { |filename| filename_to_spec_filename(filename) } + end + end.compact.uniq + + write_array_to_file(predictive_tests_pathname, matching_graphql_tests) + end + + def filter_files + changed_files.select do |filename| + filename.start_with?(*GRAPHQL_TYPES_FOLDERS.values.flatten.uniq) && + filename.end_with?('.rb') && + File.exist?(filename) + end + end + + # Regroup all GraphQL types (by edition) that are implementing another GraphQL type. + # + # The key is the type that is being implemented (e.g. NoteableInterface, TodoableInterface below) + # The value is an array of GraphQL type files that are implementing those types. + # + # Example output: + # + # { + # nil => { + # "NoteableInterface" => [ + # "app/graphql/types/alert_management/alert_type.rb", + # "app/graphql/types/design_management/design_type.rb" + # , "TodoableInterface" => [...] + # }, + # "ee" => { + # "NoteableInterface" => [ + # "app/graphql/types/alert_management/alert_type.rb", + # "app/graphql/types/design_management/design_type.rb", + # "ee/app/graphql/types/epic_type.rb"], + # "TodoableInterface"=> [...] + # } + # } + def types_hierarchies + return @types_hierarchies if @types_hierarchies + + @types_hierarchies = {} + GRAPHQL_TYPES_FOLDERS.each_key do |edition| + @types_hierarchies[edition] = Hash.new { |h, k| h[k] = [] } + + graphql_files_for_edition_glob = File.join("{#{GRAPHQL_TYPES_FOLDERS[edition].join(',')}}", '**', '*.rb') + Dir[graphql_files_for_edition_glob].each do |graphql_file| + graphql_base_types = File.read(graphql_file).scan(GRAPHQL_IMPLEMENTS_REGEXP) + next if graphql_base_types.empty? + + graphql_base_classes = graphql_base_types.flatten.map { |class_name| class_name.split('::').last } + graphql_base_classes.each do |graphql_base_class| + @types_hierarchies[edition][graphql_base_class] += [graphql_file] + end + end + end + + @types_hierarchies + end + + def filename_to_class_name(filename) + File.basename(filename, '.*').camelize + end + + def filename_to_spec_filename(filename) + spec_file = filename.sub('app', 'spec').sub('.rb', '_spec.rb') + + return spec_file if File.exist?(spec_file) + end + + private + + attr_reader :changed_files, :predictive_tests_pathname + end + end +end diff --git a/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb index e78ce266a32..b2fca3a765a 100644 --- a/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb +++ b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb @@ -9,10 +9,14 @@ require_relative '../../../../lib/gitlab_edition' module Tooling module Mappings class JsToSystemSpecsMappings < Base - def initialize(js_base_folder: 'app/assets/javascripts', system_specs_base_folder: 'spec/features') - @js_base_folder = js_base_folder - @js_base_folders = folders_for_available_editions(js_base_folder) - @system_specs_base_folder = system_specs_base_folder + def initialize( + changed_files_pathname, predictive_tests_pathname, + js_base_folder: 'app/assets/javascripts', system_specs_base_folder: 'spec/features') + @changed_files = read_array_from_file(changed_files_pathname) + @predictive_tests_pathname = predictive_tests_pathname + @js_base_folder = js_base_folder + @js_base_folders = folders_for_available_editions(js_base_folder) + @system_specs_base_folder = system_specs_base_folder # Cannot be extracted to a constant, as it depends on a variable @first_js_folder_extract_regexp = %r{ @@ -23,20 +27,22 @@ module Tooling }x end - def execute(changed_files) - filter_files(changed_files).flat_map do |edition, js_files| + def execute + matching_system_tests = filter_files.flat_map do |edition, js_files| js_keywords_regexp = Regexp.union(construct_js_keywords(js_files)) system_specs_for_edition(edition).select do |system_spec_file| system_spec_file if js_keywords_regexp.match?(system_spec_file) end end + + write_array_to_file(predictive_tests_pathname, matching_system_tests) end # Keep the files that are in the @js_base_folders folders # # Returns a hash, where the key is the GitLab edition, and the values the JS specs - def filter_files(changed_files) + def filter_files selected_files = changed_files.select do |filename| filename.start_with?(*@js_base_folders) && File.exist?(filename) end @@ -55,8 +61,12 @@ module Tooling def system_specs_for_edition(edition) all_files_in_folders_glob = File.join(@system_specs_base_folder, '**', '*') all_files_in_folders_glob = File.join(edition, all_files_in_folders_glob) if edition - Dir[all_files_in_folders_glob].select { |f| File.file?(f) } + Dir[all_files_in_folders_glob].select { |f| File.file?(f) && f.end_with?('_spec.rb') } end + + private + + attr_reader :changed_files, :predictive_tests_pathname end end end diff --git a/tooling/lib/tooling/mappings/partial_to_views_mappings.rb b/tooling/lib/tooling/mappings/partial_to_views_mappings.rb new file mode 100644 index 00000000000..8b0a5ed4ecd --- /dev/null +++ b/tooling/lib/tooling/mappings/partial_to_views_mappings.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative '../../../../lib/gitlab_edition' + +# Returns view files that include the potential rails partials from the changed files passed as input. +module Tooling + module Mappings + class PartialToViewsMappings < Base + def initialize(changed_files_pathname, views_with_partials_pathname, view_base_folder: 'app/views') + @views_with_partials_pathname = views_with_partials_pathname + @changed_files = read_array_from_file(changed_files_pathname) + @view_base_folders = folders_for_available_editions(view_base_folder) + end + + def execute + views_including_modified_partials = [] + + views_globs = view_base_folders.map { |view_base_folder| "#{view_base_folder}/**/*.html.haml" } + Dir[*views_globs].each do |view_file| + included_partial_names = find_pattern_in_file(view_file, partials_keywords_regexp) + next if included_partial_names.empty? + + included_partial_names.each do |included_partial_name| + if view_includes_modified_partial?(view_file, included_partial_name) + views_including_modified_partials << view_file + end + end + end + + write_array_to_file(views_with_partials_pathname, views_including_modified_partials) + end + + def filter_files + @_filter_files ||= changed_files.select do |filename| + filename.start_with?(*view_base_folders) && + File.basename(filename).start_with?('_') && + File.basename(filename).end_with?('.html.haml') && + File.exist?(filename) + end + end + + def partials_keywords_regexp + partial_keywords = filter_files.map do |partial_filename| + extract_partial_keyword(partial_filename) + end + + partial_regexps = partial_keywords.map do |keyword| + %r{(?:render|render_if_exists)(?: |\()(?:partial: ?)?['"]([\w\-_/]*#{keyword})['"]} + end + + Regexp.union(partial_regexps) + end + + # e.g. if app/views/clusters/clusters/_sidebar.html.haml was modified, the partial keyword is `sidebar`. + def extract_partial_keyword(partial_filename) + File.basename(partial_filename).delete_prefix('_').delete_suffix('.html.haml') + end + + # Why do we need this method? + # + # Assume app/views/clusters/clusters/_sidebar.html.haml was modified in the MR. + # + # Suppose now you find = render 'sidebar' in a view. Is this view including the sidebar partial + # that was modified, or another partial called "_sidebar.html.haml" somewhere else? + def view_includes_modified_partial?(view_file, included_partial_name) + view_file_parent_folder = File.dirname(view_file) + included_partial_filename = reconstruct_partial_filename(included_partial_name) + included_partial_relative_path = File.join(view_file_parent_folder, included_partial_filename) + + # We do this because in render (or render_if_exists) + # apparently looks for partials in other GitLab editions + # + # Example: + # + # ee/app/views/events/_epics_filter.html.haml is used in app/views/shared/_event_filter.html.haml + # with render_if_exists 'events/epics_filter' + included_partial_absolute_paths = view_base_folders.map do |view_base_folder| + File.join(view_base_folder, included_partial_filename) + end + + filter_files.include?(included_partial_relative_path) || + (filter_files & included_partial_absolute_paths).any? + end + + def reconstruct_partial_filename(partial_name) + partial_path = partial_name.split('/')[..-2] + partial_filename = partial_name.split('/').last + full_partial_filename = "_#{partial_filename}.html.haml" + + return full_partial_filename if partial_path.empty? + + File.join(partial_path.join('/'), full_partial_filename) + end + + def find_pattern_in_file(file, pattern) + File.read(file).scan(pattern).flatten.compact.uniq + end + + private + + attr_reader :changed_files, :views_with_partials_pathname, :view_base_folders + end + end +end diff --git a/tooling/lib/tooling/mappings/view_to_js_mappings.rb b/tooling/lib/tooling/mappings/view_to_js_mappings.rb index db80eb9bfe8..f2098d6acd5 100644 --- a/tooling/lib/tooling/mappings/view_to_js_mappings.rb +++ b/tooling/lib/tooling/mappings/view_to_js_mappings.rb @@ -11,37 +11,41 @@ module Tooling HTML_ATTRIBUTE_VALUE_REGEXP = /js-[-\w]+/.freeze # Search for Rails partials included in an HTML file - RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exist)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze + RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exists)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze - def initialize(view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts') - @view_base_folders = folders_for_available_editions(view_base_folder) - @js_base_folders = folders_for_available_editions(js_base_folder) + def initialize( + changed_files_pathname, predictive_tests_pathname, + view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts') + @changed_files = read_array_from_file(changed_files_pathname) + @predictive_tests_pathname = predictive_tests_pathname + @view_base_folders = folders_for_available_editions(view_base_folder) + @js_base_folders = folders_for_available_editions(js_base_folder) end - def execute(changed_files) - changed_view_files = filter_files(changed_files) - - partials = changed_view_files.flat_map do |file| + def execute + partials = filter_files.flat_map do |file| find_partials(file) end - files_to_scan = changed_view_files + partials + files_to_scan = filter_files + partials js_tags = files_to_scan.flat_map do |file| find_pattern_in_file(file, HTML_ATTRIBUTE_VALUE_REGEXP) end js_tags_regexp = Regexp.union(js_tags) - @js_base_folders.flat_map do |js_base_folder| + matching_js_files = @js_base_folders.flat_map do |js_base_folder| Dir["#{js_base_folder}/**/*.{js,vue}"].select do |js_file| file_content = File.read(js_file) js_tags_regexp.match?(file_content) end end + + write_array_to_file(predictive_tests_pathname, matching_js_files) end # Keep the files that are in the @view_base_folders folder - def filter_files(changed_files) - changed_files.select do |filename| + def filter_files + @_filter_files ||= changed_files.select do |filename| filename.start_with?(*@view_base_folders) && File.exist?(filename) end @@ -69,6 +73,10 @@ module Tooling def find_pattern_in_file(file, pattern) File.read(file).scan(pattern).flatten.uniq end + + private + + attr_reader :changed_files, :predictive_tests_pathname end end end diff --git a/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb b/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb new file mode 100644 index 00000000000..6d840dcbd71 --- /dev/null +++ b/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative '../../../../lib/gitlab_edition' + +# Returns system specs files that are related to the Rails views files that were changed in the MR. +module Tooling + module Mappings + class ViewToSystemSpecsMappings < Base + def initialize(changed_files_pathname, predictive_tests_pathname, view_base_folder: 'app/views') + @predictive_tests_pathname = predictive_tests_pathname + @changed_files = read_array_from_file(changed_files_pathname) + @view_base_folders = folders_for_available_editions(view_base_folder) + end + + def execute + found_system_specs = [] + + filter_files.each do |modified_view_file| + system_specs_exact_match = find_system_specs_exact_match(modified_view_file) + if system_specs_exact_match + found_system_specs << system_specs_exact_match + next + else + system_specs_parent_folder_match = find_system_specs_parent_folder_match(modified_view_file) + found_system_specs += system_specs_parent_folder_match unless system_specs_parent_folder_match.empty? + end + end + + write_array_to_file(predictive_tests_pathname, found_system_specs.compact.uniq.sort) + end + + private + + attr_reader :changed_files, :predictive_tests_pathname, :view_base_folders + + # Keep the views files that are in the @view_base_folders folder + def filter_files + @_filter_files ||= changed_files.select do |filename| + filename.start_with?(*view_base_folders) && + File.basename(filename).end_with?('.html.haml') && + File.exist?(filename) + end + end + + def find_system_specs_exact_match(view_file) + potential_spec_file = to_feature_spec_folder(view_file).sub('.html.haml', '_spec.rb') + + potential_spec_file if File.exist?(potential_spec_file) + end + + def find_system_specs_parent_folder_match(view_file) + parent_system_specs_folder = File.dirname(to_feature_spec_folder(view_file)) + + Dir["#{parent_system_specs_folder}/**/*_spec.rb"] + end + + # e.g. go from app/views/groups/merge_requests.html.haml to spec/features/groups/merge_requests.html.haml + def to_feature_spec_folder(view_file) + view_file.sub(%r{(ee/|jh/)?app/views}, '\1spec/features') + end + end + end +end diff --git a/tooling/lib/tooling/predictive_tests.rb b/tooling/lib/tooling/predictive_tests.rb new file mode 100644 index 00000000000..503426b5520 --- /dev/null +++ b/tooling/lib/tooling/predictive_tests.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require_relative 'find_changes' +require_relative 'find_tests' +require_relative 'mappings/graphql_base_type_mappings' +require_relative 'mappings/js_to_system_specs_mappings' +require_relative 'mappings/partial_to_views_mappings' +require_relative 'mappings/view_to_js_mappings' +require_relative 'mappings/view_to_system_specs_mappings' + +module Tooling + class PredictiveTests + REQUIRED_ENV_VARIABLES = %w[ + RSPEC_CHANGED_FILES_PATH + RSPEC_MATCHING_TESTS_PATH + RSPEC_VIEWS_INCLUDING_PARTIALS_PATH + FRONTEND_FIXTURES_MAPPING_PATH + RSPEC_MATCHING_JS_FILES_PATH + ].freeze + + def initialize + missing_env_variables = REQUIRED_ENV_VARIABLES.select { |key| ENV[key.to_s] == '' } + unless missing_env_variables.empty? + raise "[predictive tests] Missing ENV variable(s): #{missing_env_variables.join(',')}." + end + + @rspec_changed_files_path = ENV['RSPEC_CHANGED_FILES_PATH'] + @rspec_matching_tests_path = ENV['RSPEC_MATCHING_TESTS_PATH'] + @rspec_views_including_partials_path = ENV['RSPEC_VIEWS_INCLUDING_PARTIALS_PATH'] + @frontend_fixtures_mapping_path = ENV['FRONTEND_FIXTURES_MAPPING_PATH'] + @rspec_matching_js_files_path = ENV['RSPEC_MATCHING_JS_FILES_PATH'] + end + + def execute + Tooling::FindChanges.new( + from: :api, + changed_files_pathname: rspec_changed_files_path + ).execute + Tooling::FindTests.new(rspec_changed_files_path, rspec_matching_tests_path).execute + Tooling::Mappings::PartialToViewsMappings.new( + rspec_changed_files_path, rspec_views_including_partials_path).execute + Tooling::FindTests.new(rspec_views_including_partials_path, rspec_matching_tests_path).execute + Tooling::Mappings::JsToSystemSpecsMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute + Tooling::Mappings::GraphqlBaseTypeMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute + Tooling::Mappings::ViewToSystemSpecsMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute + Tooling::FindChanges.new( + from: :changed_files, + changed_files_pathname: rspec_changed_files_path, + predictive_tests_pathname: rspec_matching_tests_path, + frontend_fixtures_mapping_pathname: frontend_fixtures_mapping_path + ).execute + Tooling::Mappings::ViewToJsMappings.new(rspec_changed_files_path, rspec_matching_js_files_path).execute + end + + private + + attr_reader :rspec_changed_files_path, :rspec_matching_tests_path, :rspec_views_including_partials_path, + :frontend_fixtures_mapping_path, :rspec_matching_js_files_path + end +end diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb index eeda135f3ee..20e00763f65 100644 --- a/tooling/quality/test_level.rb +++ b/tooling/quality/test_level.rb @@ -18,6 +18,7 @@ module Quality unit: %w[ bin channels + components config contracts db @@ -54,7 +55,6 @@ module Quality views workers tooling - components ], integration: %w[ commands @@ -77,8 +77,8 @@ module Quality @patterns[level] ||= "#{prefixes_for_pattern}spec/#{folders_pattern(level)}{,/**/}*#{suffix(level)}".freeze # rubocop:disable Style/RedundantFreeze end - def regexp(level) - @regexps[level] ||= Regexp.new("#{prefixes_for_regex}spec/#{folders_regex(level)}").freeze + def regexp(level, start_with = false) + @regexps[level] ||= Regexp.new("#{'^' if start_with}#{prefixes_for_regex}spec/#{folders_regex(level)}").freeze end def level_for(file_path) |