Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-05-17 19:05:49 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-05-17 19:05:49 +0300
commit43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch)
treedceebdc68925362117480a5d672bcff122fb625b /tooling/lib
parent20c84b99005abd1c82101dfeff264ac50d2df211 (diff)
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc42
Diffstat (limited to 'tooling/lib')
-rw-r--r--tooling/lib/tooling/fast_quarantine.rb57
-rwxr-xr-xtooling/lib/tooling/find_changes.rb109
-rw-r--r--tooling/lib/tooling/find_codeowners.rb6
-rw-r--r--tooling/lib/tooling/find_files_using_feature_flags.rb48
-rw-r--r--tooling/lib/tooling/find_tests.rb31
-rw-r--r--tooling/lib/tooling/gettext_extractor.rb111
-rw-r--r--tooling/lib/tooling/helpers/file_handler.rb31
-rw-r--r--tooling/lib/tooling/helpers/predictive_tests_helper.rb (renamed from tooling/lib/tooling/mappings/base.rb)17
-rw-r--r--tooling/lib/tooling/kubernetes_client.rb141
-rw-r--r--tooling/lib/tooling/mappings/graphql_base_type_mappings.rb121
-rw-r--r--tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb33
-rw-r--r--tooling/lib/tooling/mappings/partial_to_views_mappings.rb107
-rw-r--r--tooling/lib/tooling/mappings/view_to_js_mappings.rb38
-rw-r--r--tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb66
-rw-r--r--tooling/lib/tooling/parallel_rspec_runner.rb99
-rw-r--r--tooling/lib/tooling/predictive_tests.rb62
-rw-r--r--tooling/lib/tooling/test_map_generator.rb4
17 files changed, 885 insertions, 196 deletions
diff --git a/tooling/lib/tooling/fast_quarantine.rb b/tooling/lib/tooling/fast_quarantine.rb
new file mode 100644
index 00000000000..a0dc8bc460b
--- /dev/null
+++ b/tooling/lib/tooling/fast_quarantine.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Tooling
+ class FastQuarantine
+ def initialize(fast_quarantine_path:)
+ warn "#{fast_quarantine_path} doesn't exist!" unless File.exist?(fast_quarantine_path.to_s)
+
+ @fast_quarantine_path = fast_quarantine_path
+ end
+
+ def identifiers
+ @identifiers ||= begin
+ quarantined_entity_identifiers = File.read(fast_quarantine_path).lines
+ quarantined_entity_identifiers.compact!
+ quarantined_entity_identifiers.map! do |quarantined_entity_identifier|
+ quarantined_entity_identifier.delete_prefix('./').strip
+ end
+ rescue => e # rubocop:disable Style/RescueStandardError
+ $stdout.puts e
+ []
+ end
+ end
+
+ def skip_example?(example)
+ identifiers.find do |quarantined_entity_identifier|
+ case quarantined_entity_identifier
+ when /^.+_spec\.rb\[[\d:]+\]$/ # example id, e.g. spec/tasks/gitlab/usage_data_rake_spec.rb[1:5:2:1]
+ example.id == "./#{quarantined_entity_identifier}"
+ when /^.+_spec\.rb:\d+$/ # file + line, e.g. spec/tasks/gitlab/usage_data_rake_spec.rb:42
+ fetch_metadata_from_ancestors(example, :location)
+ .any?("./#{quarantined_entity_identifier}")
+ when /^.+_spec\.rb$/ # whole file, e.g. ee/spec/features/boards/swimlanes/epics_swimlanes_sidebar_spec.rb
+ fetch_metadata_from_ancestors(example, :file_path)
+ .any?("./#{quarantined_entity_identifier}")
+ end
+ end
+ end
+
+ private
+
+ attr_reader :fast_quarantine_path
+
+ def fetch_metadata_from_ancestors(example, attribute)
+ metadata = [example.metadata[attribute]]
+ example_group = example.metadata[:example_group]
+
+ loop do
+ break if example_group.nil?
+
+ metadata << example_group[attribute]
+ example_group = example_group[:parent_example_group]
+ end
+
+ metadata
+ end
+ end
+end
diff --git a/tooling/lib/tooling/find_changes.rb b/tooling/lib/tooling/find_changes.rb
new file mode 100755
index 00000000000..c498c83d24b
--- /dev/null
+++ b/tooling/lib/tooling/find_changes.rb
@@ -0,0 +1,109 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'gitlab'
+require_relative 'helpers/predictive_tests_helper'
+
+module Tooling
+ class FindChanges
+ include Helpers::PredictiveTestsHelper
+
+ ALLOWED_FILE_TYPES = ['.js', '.vue', '.md', '.scss'].freeze
+
+ def initialize(
+ from:,
+ changed_files_pathname: nil,
+ predictive_tests_pathname: nil,
+ frontend_fixtures_mapping_pathname: nil
+ )
+
+ raise ArgumentError, ':from can only be :api or :changed_files' unless
+ %i[api changed_files].include?(from)
+
+ @gitlab_token = ENV['PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE'] || ''
+ @gitlab_endpoint = ENV['CI_API_V4_URL']
+ @mr_project_path = ENV['CI_MERGE_REQUEST_PROJECT_PATH']
+ @mr_iid = ENV['CI_MERGE_REQUEST_IID']
+ @changed_files_pathname = changed_files_pathname
+ @predictive_tests_pathname = predictive_tests_pathname
+ @frontend_fixtures_mapping_pathname = frontend_fixtures_mapping_pathname
+ @from = from
+ end
+
+ def execute
+ if changed_files_pathname.nil?
+ raise ArgumentError, "A path to the changed files file must be given as :changed_files_pathname"
+ end
+
+ case @from
+ when :api
+ write_array_to_file(changed_files_pathname, file_changes + frontend_fixture_files, append: false)
+ else
+ write_array_to_file(changed_files_pathname, frontend_fixture_files, append: true)
+ end
+ end
+
+ def only_allowed_files_changed
+ file_changes.any? && file_changes.all? { |file| ALLOWED_FILE_TYPES.include?(File.extname(file)) }
+ end
+
+ private
+
+ attr_reader :gitlab_token, :gitlab_endpoint, :mr_project_path,
+ :mr_iid, :changed_files_pathname, :predictive_tests_pathname, :frontend_fixtures_mapping_pathname
+
+ def gitlab
+ @gitlab ||= begin
+ Gitlab.configure do |config|
+ config.endpoint = gitlab_endpoint
+ config.private_token = gitlab_token
+ end
+
+ Gitlab
+ end
+ end
+
+ def add_frontend_fixture_files?
+ predictive_tests_pathname && frontend_fixtures_mapping_pathname
+ end
+
+ def frontend_fixture_files
+ # If we have a `test file -> JSON frontend fixture` mapping file, we add the files JSON frontend fixtures
+ # files to the list of changed files so that Jest can automatically run the dependent tests
+ # using --findRelatedTests flag.
+ empty = [].freeze
+
+ test_files.flat_map do |test_file|
+ frontend_fixtures_mapping[test_file] || empty
+ end
+ end
+
+ def file_changes
+ @file_changes ||=
+ case @from
+ when :api
+ mr_changes.changes.flat_map do |change|
+ change.to_h.values_at('old_path', 'new_path')
+ end.uniq
+ else
+ read_array_from_file(changed_files_pathname)
+ end
+ end
+
+ def mr_changes
+ @mr_changes ||= gitlab.merge_request_changes(mr_project_path, mr_iid)
+ end
+
+ def test_files
+ return [] if !predictive_tests_pathname || !File.exist?(predictive_tests_pathname)
+
+ read_array_from_file(predictive_tests_pathname)
+ end
+
+ def frontend_fixtures_mapping
+ return {} if !frontend_fixtures_mapping_pathname || !File.exist?(frontend_fixtures_mapping_pathname)
+
+ JSON.parse(File.read(frontend_fixtures_mapping_pathname)) # rubocop:disable Gitlab/Json
+ end
+ end
+end
diff --git a/tooling/lib/tooling/find_codeowners.rb b/tooling/lib/tooling/find_codeowners.rb
index cc37d4db1ec..e542ab9967c 100644
--- a/tooling/lib/tooling/find_codeowners.rb
+++ b/tooling/lib/tooling/find_codeowners.rb
@@ -48,11 +48,7 @@ module Tooling
def load_config
config_path = "#{__dir__}/../../config/CODEOWNERS.yml"
- if YAML.respond_to?(:safe_load_file) # Ruby 3.0+
- YAML.safe_load_file(config_path, symbolize_names: true)
- else
- YAML.safe_load(File.read(config_path), symbolize_names: true)
- end
+ YAML.safe_load_file(config_path, symbolize_names: true)
end
# Copied and modified from ee/lib/gitlab/code_owners/file.rb
diff --git a/tooling/lib/tooling/find_files_using_feature_flags.rb b/tooling/lib/tooling/find_files_using_feature_flags.rb
new file mode 100644
index 00000000000..27cace60408
--- /dev/null
+++ b/tooling/lib/tooling/find_files_using_feature_flags.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'test_file_finder'
+require_relative 'helpers/predictive_tests_helper'
+
+module Tooling
+ class FindFilesUsingFeatureFlags
+ include Helpers::PredictiveTestsHelper
+
+ def initialize(changed_files_pathname:, feature_flags_base_folder: 'config/feature_flags')
+ @changed_files_pathname = changed_files_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @feature_flags_base_folders = folders_for_available_editions(feature_flags_base_folder)
+ end
+
+ def execute
+ ff_union_regexp = Regexp.union(feature_flag_filenames)
+
+ files_using_modified_feature_flags = ruby_files.select do |ruby_file|
+ ruby_file if ff_union_regexp.match?(File.read(ruby_file))
+ end
+
+ write_array_to_file(changed_files_pathname, files_using_modified_feature_flags.uniq)
+ end
+
+ def filter_files
+ @_filter_files ||= changed_files.select do |filename|
+ filename.start_with?(*feature_flags_base_folders) &&
+ File.basename(filename).end_with?('.yml') &&
+ File.exist?(filename)
+ end
+ end
+
+ private
+
+ def feature_flag_filenames
+ filter_files.map do |feature_flag_pathname|
+ File.basename(feature_flag_pathname).delete_suffix('.yml')
+ end
+ end
+
+ def ruby_files
+ Dir["**/*.rb"].reject { |pathname| pathname.start_with?('vendor') }
+ end
+
+ attr_reader :changed_files, :changed_files_pathname, :feature_flags_base_folders
+ end
+end
diff --git a/tooling/lib/tooling/find_tests.rb b/tooling/lib/tooling/find_tests.rb
new file mode 100644
index 00000000000..bf7a608878b
--- /dev/null
+++ b/tooling/lib/tooling/find_tests.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'test_file_finder'
+require_relative 'helpers/predictive_tests_helper'
+
+module Tooling
+ class FindTests
+ include Helpers::PredictiveTestsHelper
+
+ def initialize(changed_files_pathname, predictive_tests_pathname)
+ @predictive_tests_pathname = predictive_tests_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ end
+
+ def execute
+ tff = TestFileFinder::FileFinder.new(paths: changed_files).tap do |file_finder|
+ file_finder.use TestFileFinder::MappingStrategies::PatternMatching.load('tests.yml')
+
+ if ENV['RSPEC_TESTS_MAPPING_ENABLED'] == 'true'
+ file_finder.use TestFileFinder::MappingStrategies::DirectMatching.load_json(ENV['RSPEC_TESTS_MAPPING_PATH'])
+ end
+ end
+
+ write_array_to_file(predictive_tests_pathname, tff.test_files.uniq)
+ end
+
+ private
+
+ attr_reader :changed_files, :matching_tests, :predictive_tests_pathname
+ end
+end
diff --git a/tooling/lib/tooling/gettext_extractor.rb b/tooling/lib/tooling/gettext_extractor.rb
new file mode 100644
index 00000000000..3cc05c80a2d
--- /dev/null
+++ b/tooling/lib/tooling/gettext_extractor.rb
@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'parallel'
+require 'gettext/po'
+require 'gettext/po_entry'
+require 'gettext/tools/xgettext'
+require 'gettext/tools/parser/erb'
+require 'gettext/tools/parser/ruby'
+require 'json'
+require 'open3'
+
+module Tooling
+ class GettextExtractor < GetText::Tools::XGetText
+ class HamlParser < GetText::RubyParser
+ require 'hamlit'
+ def parse_source(source)
+ super(Hamlit::Engine.new.call(source))
+ end
+ end
+
+ def initialize(
+ backend_glob: "{ee/,}{app,lib,config,locale}/**/*.{rb,erb,haml}",
+ glob_base: nil,
+ package_name: 'gitlab',
+ package_version: '1.0.0'
+ )
+ super()
+ @backend_glob = backend_glob
+ @package_name = package_name
+ @glob_base = glob_base || Dir.pwd
+ @package_version = package_version
+ # Ensure that the messages are ordered by id
+ @po_order = :msgid
+ @po_format_options = {
+ # No line breaks within a message
+ max_line_width: -1,
+ # Do not print references to files
+ include_reference_comment: false
+ }
+ end
+
+ def parse(_paths)
+ po = GetText::PO.new
+ parse_backend_files.each do |po_entry|
+ merge_po_entries(po, po_entry)
+ end
+ parse_frontend_files.each do |po_entry|
+ merge_po_entries(po, po_entry)
+ end
+ po
+ end
+
+ # Overrides method from GetText::Tools::XGetText
+ # This makes a method public and passes in an empty array of paths,
+ # as our overidden "parse" method needs no paths
+ def generate_pot
+ super([])
+ end
+
+ private
+
+ # Overrides method from GetText::Tools::XGetText
+ # in order to remove revision dates, as we check in our locale/gitlab.pot
+ def header_content
+ super.gsub(/^POT?-(?:Creation|Revision)-Date:.*\n/, '')
+ end
+
+ def merge_po_entries(po, po_entry)
+ existing_entry = po[po_entry.msgctxt, po_entry.msgid]
+ po_entry = existing_entry.merge(po_entry) if existing_entry
+
+ po[po_entry.msgctxt, po_entry.msgid] = po_entry
+ end
+
+ def parse_backend_file(path)
+ source = ::File.read(path)
+ # Do not bother parsing files not containing `_(`
+ # All of our translation helpers, _(, s_(), N_(), etc.
+ # contain it. So we can skip parsing files not containing it
+ return [] unless source.include?('_(')
+
+ case ::File.extname(path)
+ when '.rb'
+ GetText::RubyParser.new(path).parse_source(source)
+ when '.haml'
+ HamlParser.new(path).parse_source(source)
+ when '.erb'
+ GetText::ErbParser.new(path).parse
+ else
+ raise NotImplementedError
+ end
+ end
+
+ def parse_backend_files
+ files = Dir.glob(File.join(@glob_base, @backend_glob))
+ Parallel.flat_map(files) { |item| parse_backend_file(item) }
+ end
+
+ def parse_frontend_files
+ results, status = Open3.capture2('node scripts/frontend/extract_gettext_all.js --all')
+ raise StandardError, "Could not parse frontend files" unless status.success?
+
+ # rubocop:disable Gitlab/Json
+ JSON.parse(results)
+ .values
+ .flatten(1)
+ .collect { |entry| create_po_entry(*entry) }
+ # rubocop:enable Gitlab/Json
+ end
+ end
+end
diff --git a/tooling/lib/tooling/helpers/file_handler.rb b/tooling/lib/tooling/helpers/file_handler.rb
new file mode 100644
index 00000000000..88248e31df2
--- /dev/null
+++ b/tooling/lib/tooling/helpers/file_handler.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module Tooling
+ module Helpers
+ module FileHandler
+ def read_array_from_file(file)
+ FileUtils.touch file
+
+ File.read(file).split(' ')
+ end
+
+ def write_array_to_file(file, content_array, append: true)
+ FileUtils.touch file
+
+ # We sort the array to make it easier to read the output file
+ content_array.sort!
+
+ output_content =
+ if append
+ [File.read(file), *content_array].join(' ').lstrip
+ else
+ content_array.join(' ')
+ end
+
+ File.write(file, output_content)
+ end
+ end
+ end
+end
diff --git a/tooling/lib/tooling/mappings/base.rb b/tooling/lib/tooling/helpers/predictive_tests_helper.rb
index 93d3a967114..b8e5a30024e 100644
--- a/tooling/lib/tooling/mappings/base.rb
+++ b/tooling/lib/tooling/helpers/predictive_tests_helper.rb
@@ -1,22 +1,13 @@
# frozen_string_literal: true
require_relative '../../../../lib/gitlab_edition'
+require_relative '../helpers/file_handler'
# Returns system specs files that are related to the JS files that were changed in the MR.
module Tooling
- module Mappings
- class Base
- # Input: A list of space-separated files
- # Output: A list of space-separated specs files (JS, Ruby, ...)
- def execute(changed_files)
- raise "Not Implemented"
- end
-
- # Input: A list of space-separated files
- # Output: array/hash of files
- def filter_files(changed_files)
- raise "Not Implemented"
- end
+ module Helpers
+ module PredictiveTestsHelper
+ include FileHandler
# Input: A folder
# Output: An array of folders, each prefixed with a GitLab edition
diff --git a/tooling/lib/tooling/kubernetes_client.rb b/tooling/lib/tooling/kubernetes_client.rb
index ab914db5777..5579f130a84 100644
--- a/tooling/lib/tooling/kubernetes_client.rb
+++ b/tooling/lib/tooling/kubernetes_client.rb
@@ -6,72 +6,23 @@ require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen)
module Tooling
class KubernetesClient
- RESOURCE_LIST = 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd'
K8S_ALLOWED_NAMESPACES_REGEX = /^review-(?!apps).+/.freeze
CommandFailedError = Class.new(StandardError)
- attr_reader :namespace
+ def cleanup_namespaces_by_created_at(created_before:)
+ stale_namespaces = namespaces_created_before(created_before: created_before)
- def initialize(namespace:)
- @namespace = namespace
- end
-
- def cleanup_by_release(release_name:, wait: true)
- delete_by_selector(release_name: release_name, wait: wait)
- delete_by_matching_name(release_name: release_name)
- end
-
- def cleanup_by_created_at(resource_type:, created_before:, wait: true)
- resource_names = resource_names_created_before(resource_type: resource_type, created_before: created_before)
- return if resource_names.empty?
-
- delete_by_exact_names(resource_type: resource_type, resource_names: resource_names, wait: wait)
- end
-
- def cleanup_review_app_namespaces(created_before:, wait: true)
- namespaces = review_app_namespaces_created_before(created_before: created_before)
- return if namespaces.empty?
-
- delete_namespaces_by_exact_names(resource_names: namespaces, wait: wait)
- end
-
- private
-
- def delete_by_selector(release_name:, wait:)
- selector = case release_name
- when String
- %(-l release="#{release_name}")
- when Array
- %(-l 'release in (#{release_name.join(', ')})')
- else
- raise ArgumentError, 'release_name must be a string or an array'
- end
-
- command = [
- 'delete',
- RESOURCE_LIST,
- %(--namespace "#{namespace}"),
- '--now',
- '--ignore-not-found',
- %(--wait=#{wait}),
- selector
- ]
+ # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby.
+ review_apps_stale_namespaces = stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) }
+ return if review_apps_stale_namespaces.empty?
- run_command(command)
+ delete_namespaces(review_apps_stale_namespaces)
end
- def delete_by_exact_names(resource_names:, wait:, resource_type: nil)
- command = [
- 'delete',
- resource_type,
- %(--namespace "#{namespace}"),
- '--now',
- '--ignore-not-found',
- %(--wait=#{wait}),
- resource_names.join(' ')
- ]
+ def delete_namespaces(namespaces)
+ return if namespaces.any? { |ns| !K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) }
- run_command(command)
+ run_command("kubectl delete namespace --now --ignore-not-found #{namespaces.join(' ')}")
end
def delete_namespaces_by_exact_names(resource_names:, wait:)
@@ -87,87 +38,29 @@ module Tooling
run_command(command)
end
- def delete_by_matching_name(release_name:)
- resource_names = raw_resource_names
- command = [
- 'delete',
- %(--namespace "#{namespace}"),
- '--ignore-not-found'
- ]
-
- Array(release_name).each do |release|
- resource_names
- .select { |resource_name| resource_name.include?(release) }
- .each { |matching_resource| run_command(command + [matching_resource]) }
- end
- end
-
- def raw_resource_names
- command = [
- 'get',
- RESOURCE_LIST,
- %(--namespace "#{namespace}"),
- '-o name'
- ]
- run_command(command).lines.map(&:strip)
- end
-
- def resource_names_created_before(resource_type:, created_before:)
- command = [
- 'get',
- resource_type,
- %(--namespace "#{namespace}"),
- "--sort-by='{.metadata.creationTimestamp}'",
- '-o json'
- ]
+ def namespaces_created_before(created_before:)
+ response = run_command("kubectl get namespace --all-namespaces --sort-by='{.metadata.creationTimestamp}' -o json")
- response = run_command(command)
-
- resources_created_before_date(response, created_before)
- end
-
- def review_app_namespaces_created_before(created_before:)
- command = [
- 'get',
- 'namespace',
- "--sort-by='{.metadata.creationTimestamp}'",
- '-o json'
- ]
-
- response = run_command(command)
-
- stale_namespaces = resources_created_before_date(response, created_before)
-
- # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby.
- stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) }
- end
-
- def resources_created_before_date(response, date)
items = JSON.parse(response)['items'] # rubocop:disable Gitlab/Json
-
- items.each_with_object([]) do |item, result|
+ items.filter_map do |item|
item_created_at = Time.parse(item.dig('metadata', 'creationTimestamp'))
- if item_created_at < date
- resource_name = item.dig('metadata', 'name')
- result << resource_name
- end
+ item.dig('metadata', 'name') if item_created_at < created_before
end
rescue ::JSON::ParserError => ex
- puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output
+ puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}"
[]
end
def run_command(command)
- final_command = ['kubectl', *command.compact].join(' ')
- puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
+ puts "Running command: `#{command}`"
- result = Gitlab::Popen.popen_with_detail([final_command])
+ result = Gitlab::Popen.popen_with_detail([command])
if result.status.success?
result.stdout.chomp.freeze
else
- raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
+ raise CommandFailedError, "The `#{command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
end
end
end
diff --git a/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb b/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb
new file mode 100644
index 00000000000..80aa99efc96
--- /dev/null
+++ b/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb
@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require 'active_support/inflector'
+
+require_relative '../helpers/predictive_tests_helper'
+require_relative '../../../../lib/gitlab_edition'
+
+# If a GraphQL type class changed, we try to identify the other GraphQL types that potentially include this type.
+module Tooling
+ module Mappings
+ class GraphqlBaseTypeMappings
+ include Helpers::PredictiveTestsHelper
+
+ # Checks for the implements keyword, and graphql_base_types the class name
+ GRAPHQL_IMPLEMENTS_REGEXP = /implements[( ]([\w:]+)[)]?$/
+
+ # GraphQL types are a bit scattered in the codebase based on the edition.
+ #
+ # Also, a higher edition is able to include lower editions.
+ # e.g. EE can include FOSS GraphQL types, and JH can include all GraphQL types
+ GRAPHQL_TYPES_FOLDERS_FOSS = ['app/graphql/types'].freeze
+ GRAPHQL_TYPES_FOLDERS_EE = GRAPHQL_TYPES_FOLDERS_FOSS + ['ee/app/graphql/types', 'ee/app/graphql/ee/types']
+ GRAPHQL_TYPES_FOLDERS_JH = GRAPHQL_TYPES_FOLDERS_EE + ['jh/app/graphql/types', 'jh/app/graphql/jh/types']
+ GRAPHQL_TYPES_FOLDERS = {
+ nil => GRAPHQL_TYPES_FOLDERS_FOSS,
+ 'ee' => GRAPHQL_TYPES_FOLDERS_EE,
+ 'jh' => GRAPHQL_TYPES_FOLDERS_JH
+ }.freeze
+
+ def initialize(changed_files_pathname, predictive_tests_pathname)
+ @predictive_tests_pathname = predictive_tests_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ end
+
+ def execute
+ # We go through the available editions when searching for base types
+ #
+ # `nil` is the FOSS edition
+ matching_graphql_tests = ([nil] + ::GitlabEdition.extensions).flat_map do |edition|
+ hierarchy = types_hierarchies[edition]
+
+ filter_files.flat_map do |graphql_file|
+ children_types = hierarchy[filename_to_class_name(graphql_file)]
+ next if children_types.empty?
+
+ # We find the specs for the children GraphQL types that are implementing the current GraphQL Type
+ children_types.map { |filename| filename_to_spec_filename(filename) }
+ end
+ end.compact.uniq
+
+ write_array_to_file(predictive_tests_pathname, matching_graphql_tests)
+ end
+
+ def filter_files
+ changed_files.select do |filename|
+ filename.start_with?(*GRAPHQL_TYPES_FOLDERS.values.flatten.uniq) &&
+ filename.end_with?('.rb') &&
+ File.exist?(filename)
+ end
+ end
+
+ # Regroup all GraphQL types (by edition) that are implementing another GraphQL type.
+ #
+ # The key is the type that is being implemented (e.g. NoteableInterface, TodoableInterface below)
+ # The value is an array of GraphQL type files that are implementing those types.
+ #
+ # Example output:
+ #
+ # {
+ # nil => {
+ # "NoteableInterface" => [
+ # "app/graphql/types/alert_management/alert_type.rb",
+ # "app/graphql/types/design_management/design_type.rb"
+ # , "TodoableInterface" => [...]
+ # },
+ # "ee" => {
+ # "NoteableInterface" => [
+ # "app/graphql/types/alert_management/alert_type.rb",
+ # "app/graphql/types/design_management/design_type.rb",
+ # "ee/app/graphql/types/epic_type.rb"],
+ # "TodoableInterface"=> [...]
+ # }
+ # }
+ def types_hierarchies
+ return @types_hierarchies if @types_hierarchies
+
+ @types_hierarchies = {}
+ GRAPHQL_TYPES_FOLDERS.each_key do |edition|
+ @types_hierarchies[edition] = Hash.new { |h, k| h[k] = [] }
+
+ graphql_files_for_edition_glob = File.join("{#{GRAPHQL_TYPES_FOLDERS[edition].join(',')}}", '**', '*.rb')
+ Dir[graphql_files_for_edition_glob].each do |graphql_file|
+ graphql_base_types = File.read(graphql_file).scan(GRAPHQL_IMPLEMENTS_REGEXP)
+ next if graphql_base_types.empty?
+
+ graphql_base_classes = graphql_base_types.flatten.map { |class_name| class_name.split('::').last }
+ graphql_base_classes.each do |graphql_base_class|
+ @types_hierarchies[edition][graphql_base_class] += [graphql_file]
+ end
+ end
+ end
+
+ @types_hierarchies
+ end
+
+ def filename_to_class_name(filename)
+ File.basename(filename, '.*').camelize
+ end
+
+ def filename_to_spec_filename(filename)
+ spec_file = filename.sub('app', 'spec').sub('.rb', '_spec.rb')
+
+ return spec_file if File.exist?(spec_file)
+ end
+
+ private
+
+ attr_reader :changed_files, :predictive_tests_pathname
+ end
+ end
+end
diff --git a/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb
index 365e466011b..bc2cd259fdc 100644
--- a/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb
+++ b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb
@@ -2,40 +2,49 @@
require 'active_support/inflector'
-require_relative 'base'
+require_relative '../helpers/predictive_tests_helper'
require_relative '../../../../lib/gitlab_edition'
# Returns system specs files that are related to the JS files that were changed in the MR.
module Tooling
module Mappings
- class JsToSystemSpecsMappings < Base
- def initialize(js_base_folder: 'app/assets/javascripts', system_specs_base_folder: 'spec/features')
- @js_base_folder = js_base_folder
- @js_base_folders = folders_for_available_editions(js_base_folder)
- @system_specs_base_folder = system_specs_base_folder
+ class JsToSystemSpecsMappings
+ include Helpers::PredictiveTestsHelper
+
+ def initialize(
+ changed_files_pathname, predictive_tests_pathname,
+ js_base_folder: 'app/assets/javascripts', system_specs_base_folder: 'spec/features')
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @predictive_tests_pathname = predictive_tests_pathname
+ @js_base_folder = js_base_folder
+ @js_base_folders = folders_for_available_editions(js_base_folder)
+ @system_specs_base_folder = system_specs_base_folder
# Cannot be extracted to a constant, as it depends on a variable
@first_js_folder_extract_regexp = %r{
(?:.*/)? # Skips the GitLab edition (e.g. ee/, jh/)
#{@js_base_folder}/ # Most likely app/assets/javascripts/
+ (?:pages/)? # If under a pages folder, we capture the following folder
([\w-]*) # Captures the first folder
}x
end
- def execute(changed_files)
- filter_files(changed_files).flat_map do |edition, js_files|
+ def execute
+ matching_system_tests = filter_files.flat_map do |edition, js_files|
js_keywords_regexp = Regexp.union(construct_js_keywords(js_files))
system_specs_for_edition(edition).select do |system_spec_file|
system_spec_file if js_keywords_regexp.match?(system_spec_file)
end
end
+
+ write_array_to_file(predictive_tests_pathname, matching_system_tests)
end
# Keep the files that are in the @js_base_folders folders
#
# Returns a hash, where the key is the GitLab edition, and the values the JS specs
- def filter_files(changed_files)
+ def filter_files
selected_files = changed_files.select do |filename|
filename.start_with?(*@js_base_folders) && File.exist?(filename)
end
@@ -54,8 +63,12 @@ module Tooling
def system_specs_for_edition(edition)
all_files_in_folders_glob = File.join(@system_specs_base_folder, '**', '*')
all_files_in_folders_glob = File.join(edition, all_files_in_folders_glob) if edition
- Dir[all_files_in_folders_glob].select { |f| File.file?(f) }
+ Dir[all_files_in_folders_glob].select { |f| File.file?(f) && f.end_with?('_spec.rb') }
end
+
+ private
+
+ attr_reader :changed_files, :predictive_tests_pathname
end
end
end
diff --git a/tooling/lib/tooling/mappings/partial_to_views_mappings.rb b/tooling/lib/tooling/mappings/partial_to_views_mappings.rb
new file mode 100644
index 00000000000..931cacea77f
--- /dev/null
+++ b/tooling/lib/tooling/mappings/partial_to_views_mappings.rb
@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require_relative '../helpers/predictive_tests_helper'
+require_relative '../../../../lib/gitlab_edition'
+
+# Returns view files that include the potential rails partials from the changed files passed as input.
+module Tooling
+ module Mappings
+ class PartialToViewsMappings
+ include Helpers::PredictiveTestsHelper
+
+ def initialize(changed_files_pathname, views_with_partials_pathname, view_base_folder: 'app/views')
+ @views_with_partials_pathname = views_with_partials_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @view_base_folders = folders_for_available_editions(view_base_folder)
+ end
+
+ def execute
+ views_including_modified_partials = []
+
+ views_globs = view_base_folders.map { |view_base_folder| "#{view_base_folder}/**/*.html.haml" }
+ Dir[*views_globs].each do |view_file|
+ included_partial_names = find_pattern_in_file(view_file, partials_keywords_regexp)
+ next if included_partial_names.empty?
+
+ included_partial_names.each do |included_partial_name|
+ if view_includes_modified_partial?(view_file, included_partial_name)
+ views_including_modified_partials << view_file
+ end
+ end
+ end
+
+ write_array_to_file(views_with_partials_pathname, views_including_modified_partials)
+ end
+
+ def filter_files
+ @_filter_files ||= changed_files.select do |filename|
+ filename.start_with?(*view_base_folders) &&
+ File.basename(filename).start_with?('_') &&
+ File.basename(filename).end_with?('.html.haml') &&
+ File.exist?(filename)
+ end
+ end
+
+ def partials_keywords_regexp
+ partial_keywords = filter_files.map do |partial_filename|
+ extract_partial_keyword(partial_filename)
+ end
+
+ partial_regexps = partial_keywords.map do |keyword|
+ %r{(?:render|render_if_exists)(?: |\()(?:partial: ?)?['"]([\w\-_/]*#{keyword})['"]}
+ end
+
+ Regexp.union(partial_regexps)
+ end
+
+ # e.g. if app/views/clusters/clusters/_sidebar.html.haml was modified, the partial keyword is `sidebar`.
+ def extract_partial_keyword(partial_filename)
+ File.basename(partial_filename).delete_prefix('_').delete_suffix('.html.haml')
+ end
+
+ # Why do we need this method?
+ #
+ # Assume app/views/clusters/clusters/_sidebar.html.haml was modified in the MR.
+ #
+ # Suppose now you find = render 'sidebar' in a view. Is this view including the sidebar partial
+ # that was modified, or another partial called "_sidebar.html.haml" somewhere else?
+ def view_includes_modified_partial?(view_file, included_partial_name)
+ view_file_parent_folder = File.dirname(view_file)
+ included_partial_filename = reconstruct_partial_filename(included_partial_name)
+ included_partial_relative_path = File.join(view_file_parent_folder, included_partial_filename)
+
+ # We do this because in render (or render_if_exists)
+ # apparently looks for partials in other GitLab editions
+ #
+ # Example:
+ #
+ # ee/app/views/events/_epics_filter.html.haml is used in app/views/shared/_event_filter.html.haml
+ # with render_if_exists 'events/epics_filter'
+ included_partial_absolute_paths = view_base_folders.map do |view_base_folder|
+ File.join(view_base_folder, included_partial_filename)
+ end
+
+ filter_files.include?(included_partial_relative_path) ||
+ (filter_files & included_partial_absolute_paths).any?
+ end
+
+ def reconstruct_partial_filename(partial_name)
+ partial_path = partial_name.split('/')[..-2]
+ partial_filename = partial_name.split('/').last
+ full_partial_filename = "_#{partial_filename}.html.haml"
+
+ return full_partial_filename if partial_path.empty?
+
+ File.join(partial_path.join('/'), full_partial_filename)
+ end
+
+ def find_pattern_in_file(file, pattern)
+ File.read(file).scan(pattern).flatten.compact.uniq
+ end
+
+ private
+
+ attr_reader :changed_files, :views_with_partials_pathname, :view_base_folders
+ end
+ end
+end
diff --git a/tooling/lib/tooling/mappings/view_to_js_mappings.rb b/tooling/lib/tooling/mappings/view_to_js_mappings.rb
index db80eb9bfe8..b78c354f9d2 100644
--- a/tooling/lib/tooling/mappings/view_to_js_mappings.rb
+++ b/tooling/lib/tooling/mappings/view_to_js_mappings.rb
@@ -1,47 +1,53 @@
# frozen_string_literal: true
-require_relative 'base'
+require_relative '../helpers/predictive_tests_helper'
require_relative '../../../../lib/gitlab_edition'
# Returns JS files that are related to the Rails views files that were changed in the MR.
module Tooling
module Mappings
- class ViewToJsMappings < Base
+ class ViewToJsMappings
+ include Helpers::PredictiveTestsHelper
+
# The HTML attribute value pattern we're looking for to match an HTML file to a JS file.
HTML_ATTRIBUTE_VALUE_REGEXP = /js-[-\w]+/.freeze
# Search for Rails partials included in an HTML file
- RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exist)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze
+ RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exists)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze
- def initialize(view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts')
- @view_base_folders = folders_for_available_editions(view_base_folder)
- @js_base_folders = folders_for_available_editions(js_base_folder)
+ def initialize(
+ changed_files_pathname, predictive_tests_pathname,
+ view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts')
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @predictive_tests_pathname = predictive_tests_pathname
+ @view_base_folders = folders_for_available_editions(view_base_folder)
+ @js_base_folders = folders_for_available_editions(js_base_folder)
end
- def execute(changed_files)
- changed_view_files = filter_files(changed_files)
-
- partials = changed_view_files.flat_map do |file|
+ def execute
+ partials = filter_files.flat_map do |file|
find_partials(file)
end
- files_to_scan = changed_view_files + partials
+ files_to_scan = filter_files + partials
js_tags = files_to_scan.flat_map do |file|
find_pattern_in_file(file, HTML_ATTRIBUTE_VALUE_REGEXP)
end
js_tags_regexp = Regexp.union(js_tags)
- @js_base_folders.flat_map do |js_base_folder|
+ matching_js_files = @js_base_folders.flat_map do |js_base_folder|
Dir["#{js_base_folder}/**/*.{js,vue}"].select do |js_file|
file_content = File.read(js_file)
js_tags_regexp.match?(file_content)
end
end
+
+ write_array_to_file(predictive_tests_pathname, matching_js_files)
end
# Keep the files that are in the @view_base_folders folder
- def filter_files(changed_files)
- changed_files.select do |filename|
+ def filter_files
+ @_filter_files ||= changed_files.select do |filename|
filename.start_with?(*@view_base_folders) &&
File.exist?(filename)
end
@@ -69,6 +75,10 @@ module Tooling
def find_pattern_in_file(file, pattern)
File.read(file).scan(pattern).flatten.uniq
end
+
+ private
+
+ attr_reader :changed_files, :predictive_tests_pathname
end
end
end
diff --git a/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb b/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb
new file mode 100644
index 00000000000..1542c817745
--- /dev/null
+++ b/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative '../helpers/predictive_tests_helper'
+require_relative '../../../../lib/gitlab_edition'
+
+# Returns system specs files that are related to the Rails views files that were changed in the MR.
+module Tooling
+ module Mappings
+ class ViewToSystemSpecsMappings
+ include Helpers::PredictiveTestsHelper
+
+ def initialize(changed_files_pathname, predictive_tests_pathname, view_base_folder: 'app/views')
+ @predictive_tests_pathname = predictive_tests_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @view_base_folders = folders_for_available_editions(view_base_folder)
+ end
+
+ def execute
+ found_system_specs = []
+
+ filter_files.each do |modified_view_file|
+ system_specs_exact_match = find_system_specs_exact_match(modified_view_file)
+ if system_specs_exact_match
+ found_system_specs << system_specs_exact_match
+ next
+ else
+ system_specs_parent_folder_match = find_system_specs_parent_folder_match(modified_view_file)
+ found_system_specs += system_specs_parent_folder_match unless system_specs_parent_folder_match.empty?
+ end
+ end
+
+ write_array_to_file(predictive_tests_pathname, found_system_specs.compact.uniq.sort)
+ end
+
+ private
+
+ attr_reader :changed_files, :predictive_tests_pathname, :view_base_folders
+
+ # Keep the views files that are in the @view_base_folders folder
+ def filter_files
+ @_filter_files ||= changed_files.select do |filename|
+ filename.start_with?(*view_base_folders) &&
+ File.basename(filename).end_with?('.html.haml') &&
+ File.exist?(filename)
+ end
+ end
+
+ def find_system_specs_exact_match(view_file)
+ potential_spec_file = to_feature_spec_folder(view_file).sub('.html.haml', '_spec.rb')
+
+ potential_spec_file if File.exist?(potential_spec_file)
+ end
+
+ def find_system_specs_parent_folder_match(view_file)
+ parent_system_specs_folder = File.dirname(to_feature_spec_folder(view_file))
+
+ Dir["#{parent_system_specs_folder}/**/*_spec.rb"]
+ end
+
+ # e.g. go from app/views/groups/merge_requests.html.haml to spec/features/groups/merge_requests.html.haml
+ def to_feature_spec_folder(view_file)
+ view_file.sub(%r{(ee/|jh/)?app/views}, '\1spec/features')
+ end
+ end
+ end
+end
diff --git a/tooling/lib/tooling/parallel_rspec_runner.rb b/tooling/lib/tooling/parallel_rspec_runner.rb
index b1ddc91e831..834d9ec23a7 100644
--- a/tooling/lib/tooling/parallel_rspec_runner.rb
+++ b/tooling/lib/tooling/parallel_rspec_runner.rb
@@ -1,6 +1,60 @@
# frozen_string_literal: true
require 'knapsack'
+require 'fileutils'
+
+module Knapsack
+ module Distributors
+ class BaseDistributor
+ # Refine https://github.com/KnapsackPro/knapsack/blob/v1.21.1/lib/knapsack/distributors/base_distributor.rb
+ # to take in account the additional filtering we do for predictive jobs.
+ module BaseDistributorWithTestFiltering
+ attr_reader :filter_tests
+
+ def initialize(args = {})
+ super
+
+ @filter_tests = args[:filter_tests]
+ end
+
+ def all_tests
+ @all_tests ||= begin
+ pattern_tests = Dir.glob(test_file_pattern).uniq
+
+ if filter_tests.empty?
+ pattern_tests
+ else
+ pattern_tests & filter_tests
+ end
+ end.sort
+ end
+ end
+
+ prepend BaseDistributorWithTestFiltering
+ end
+ end
+
+ class AllocatorBuilder
+ # Refine https://github.com/KnapsackPro/knapsack/blob/v1.21.1/lib/knapsack/allocator_builder.rb
+ # to take in account the additional filtering we do for predictive jobs.
+ module AllocatorBuilderWithTestFiltering
+ attr_accessor :filter_tests
+
+ def allocator
+ Knapsack::Allocator.new({
+ report: Knapsack.report.open,
+ test_file_pattern: test_file_pattern,
+ ci_node_total: Knapsack::Config::Env.ci_node_total,
+ ci_node_index: Knapsack::Config::Env.ci_node_index,
+ # Additional argument
+ filter_tests: filter_tests
+ })
+ end
+ end
+
+ prepend AllocatorBuilderWithTestFiltering
+ end
+end
# A custom parallel rspec runner based on Knapsack runner
# which takes in additional option for a file containing
@@ -13,32 +67,26 @@ require 'knapsack'
# would be executed in the CI node.
#
# Reference:
-# https://github.com/ArturT/knapsack/blob/v1.20.0/lib/knapsack/runners/rspec_runner.rb
+# https://github.com/ArturT/knapsack/blob/v1.21.1/lib/knapsack/runners/rspec_runner.rb
module Tooling
class ParallelRSpecRunner
def self.run(rspec_args: nil, filter_tests_file: nil)
new(rspec_args: rspec_args, filter_tests_file: filter_tests_file).run
end
- def initialize(allocator: knapsack_allocator, filter_tests_file: nil, rspec_args: nil)
- @allocator = allocator
+ def initialize(filter_tests_file: nil, rspec_args: nil)
@filter_tests_file = filter_tests_file
@rspec_args = rspec_args&.split(' ') || []
end
def run
- Knapsack.logger.info
- Knapsack.logger.info 'Knapsack node specs:'
- Knapsack.logger.info node_tests
- Knapsack.logger.info
- Knapsack.logger.info 'Filter specs:'
- Knapsack.logger.info filter_tests
- Knapsack.logger.info
- Knapsack.logger.info 'Running specs:'
- Knapsack.logger.info tests_to_run
- Knapsack.logger.info
-
- if tests_to_run.empty?
+ if ENV['KNAPSACK_RSPEC_SUITE_REPORT_PATH']
+ knapsack_dir = File.dirname(ENV['KNAPSACK_RSPEC_SUITE_REPORT_PATH'])
+ FileUtils.mkdir_p(knapsack_dir)
+ File.write(File.join(knapsack_dir, 'node_specs.txt'), node_tests.join("\n"))
+ end
+
+ if node_tests.empty?
Knapsack.logger.info 'No tests to run on this node, exiting.'
return
end
@@ -50,26 +98,16 @@ module Tooling
private
- attr_reader :allocator, :filter_tests_file, :rspec_args
+ attr_reader :filter_tests_file, :rspec_args
def rspec_command
%w[bundle exec rspec].tap do |cmd|
cmd.push(*rspec_args)
- cmd.push('--default-path', allocator.test_dir)
cmd.push('--')
- cmd.push(*tests_to_run)
+ cmd.push(*node_tests)
end
end
- def tests_to_run
- if filter_tests.empty?
- Knapsack.logger.info 'Running all node tests without filter'
- return node_tests
- end
-
- @tests_to_run ||= node_tests & filter_tests
- end
-
def node_tests
allocator.node_tests
end
@@ -85,8 +123,11 @@ module Tooling
File.read(filter_tests_file).split(' ')
end
- def knapsack_allocator
- Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator
+ def allocator
+ @allocator ||=
+ Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).tap do |builder|
+ builder.filter_tests = filter_tests
+ end.allocator
end
end
end
diff --git a/tooling/lib/tooling/predictive_tests.rb b/tooling/lib/tooling/predictive_tests.rb
new file mode 100644
index 00000000000..1ad63e111e3
--- /dev/null
+++ b/tooling/lib/tooling/predictive_tests.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative 'find_changes'
+require_relative 'find_tests'
+require_relative 'find_files_using_feature_flags'
+require_relative 'mappings/graphql_base_type_mappings'
+require_relative 'mappings/js_to_system_specs_mappings'
+require_relative 'mappings/partial_to_views_mappings'
+require_relative 'mappings/view_to_js_mappings'
+require_relative 'mappings/view_to_system_specs_mappings'
+
+module Tooling
+ class PredictiveTests
+ REQUIRED_ENV_VARIABLES = %w[
+ RSPEC_CHANGED_FILES_PATH
+ RSPEC_MATCHING_TESTS_PATH
+ RSPEC_VIEWS_INCLUDING_PARTIALS_PATH
+ FRONTEND_FIXTURES_MAPPING_PATH
+ RSPEC_MATCHING_JS_FILES_PATH
+ ].freeze
+
+ def initialize
+ missing_env_variables = REQUIRED_ENV_VARIABLES.select { |key| ENV[key.to_s] == '' }
+ unless missing_env_variables.empty?
+ raise "[predictive tests] Missing ENV variable(s): #{missing_env_variables.join(',')}."
+ end
+
+ @rspec_changed_files_path = ENV['RSPEC_CHANGED_FILES_PATH']
+ @rspec_matching_tests_path = ENV['RSPEC_MATCHING_TESTS_PATH']
+ @rspec_views_including_partials_path = ENV['RSPEC_VIEWS_INCLUDING_PARTIALS_PATH']
+ @frontend_fixtures_mapping_path = ENV['FRONTEND_FIXTURES_MAPPING_PATH']
+ @rspec_matching_js_files_path = ENV['RSPEC_MATCHING_JS_FILES_PATH']
+ end
+
+ def execute
+ Tooling::FindChanges.new(
+ from: :api,
+ changed_files_pathname: rspec_changed_files_path
+ ).execute
+ Tooling::FindFilesUsingFeatureFlags.new(changed_files_pathname: rspec_changed_files_path).execute
+ Tooling::FindTests.new(rspec_changed_files_path, rspec_matching_tests_path).execute
+ Tooling::Mappings::PartialToViewsMappings.new(
+ rspec_changed_files_path, rspec_views_including_partials_path).execute
+ Tooling::FindTests.new(rspec_views_including_partials_path, rspec_matching_tests_path).execute
+ Tooling::Mappings::JsToSystemSpecsMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute
+ Tooling::Mappings::GraphqlBaseTypeMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute
+ Tooling::Mappings::ViewToSystemSpecsMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute
+ Tooling::FindChanges.new(
+ from: :changed_files,
+ changed_files_pathname: rspec_changed_files_path,
+ predictive_tests_pathname: rspec_matching_tests_path,
+ frontend_fixtures_mapping_pathname: frontend_fixtures_mapping_path
+ ).execute
+ Tooling::Mappings::ViewToJsMappings.new(rspec_changed_files_path, rspec_matching_js_files_path).execute
+ end
+
+ private
+
+ attr_reader :rspec_changed_files_path, :rspec_matching_tests_path, :rspec_views_including_partials_path,
+ :frontend_fixtures_mapping_path, :rspec_matching_js_files_path
+ end
+end
diff --git a/tooling/lib/tooling/test_map_generator.rb b/tooling/lib/tooling/test_map_generator.rb
index f96f33ff074..88b4353b232 100644
--- a/tooling/lib/tooling/test_map_generator.rb
+++ b/tooling/lib/tooling/test_map_generator.rb
@@ -12,7 +12,9 @@ module Tooling
def parse(yaml_files)
Array(yaml_files).each do |yaml_file|
data = File.read(yaml_file)
- metadata, example_groups = data.split("---\n").reject(&:empty?).map { |yml| YAML.safe_load(yml, [Symbol]) }
+ metadata, example_groups = data.split("---\n").reject(&:empty?).map do |yml|
+ YAML.safe_load(yml, permitted_classes: [Symbol])
+ end
if example_groups.nil?
puts "No examples in #{yaml_file}! Metadata: #{metadata}"