Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-04-20 14:43:17 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-04-20 14:43:17 +0300
commitdfc94207fec2d84314b1a5410cface22e8b369bd (patch)
treec54022f61ced104305889a64de080998a0dc773b /tooling
parentb874efeff674f6bf0355d5d242ecf81c6f7155df (diff)
Add latest changes from gitlab-org/gitlab@15-11-stable-eev15.11.0-rc42
Diffstat (limited to 'tooling')
-rwxr-xr-xtooling/bin/find_changes82
-rwxr-xr-xtooling/bin/find_only_js_changes12
-rwxr-xr-xtooling/bin/find_tests19
-rwxr-xr-xtooling/bin/gettext_extractor29
-rwxr-xr-xtooling/bin/js_to_system_specs_mappings14
-rwxr-xr-xtooling/bin/predictive_tests6
-rwxr-xr-xtooling/bin/view_to_js_mappings10
-rw-r--r--tooling/danger/feature_flag.rb6
-rw-r--r--tooling/danger/multiversion.rb35
-rw-r--r--tooling/danger/specs.rb88
-rw-r--r--tooling/danger/specs/feature_category_suggestion.rb42
-rw-r--r--tooling/danger/specs/match_with_array_suggestion.rb17
-rw-r--r--tooling/danger/specs/project_factory_suggestion.rb38
-rw-r--r--tooling/danger/stable_branch.rb4
-rw-r--r--tooling/danger/suggestion.rb39
-rw-r--r--tooling/docs/deprecation_handling.rb7
-rwxr-xr-xtooling/lib/tooling/find_changes.rb107
-rw-r--r--tooling/lib/tooling/find_tests.rb31
-rw-r--r--tooling/lib/tooling/gettext_extractor.rb106
-rw-r--r--tooling/lib/tooling/helpers/file_handler.rb31
-rw-r--r--tooling/lib/tooling/kubernetes_client.rb141
-rw-r--r--tooling/lib/tooling/mappings/base.rb13
-rw-r--r--tooling/lib/tooling/mappings/graphql_base_type_mappings.rb119
-rw-r--r--tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb26
-rw-r--r--tooling/lib/tooling/mappings/partial_to_views_mappings.rb105
-rw-r--r--tooling/lib/tooling/mappings/view_to_js_mappings.rb32
-rw-r--r--tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb64
-rw-r--r--tooling/lib/tooling/predictive_tests.rb60
-rw-r--r--tooling/quality/test_level.rb6
29 files changed, 919 insertions, 370 deletions
diff --git a/tooling/bin/find_changes b/tooling/bin/find_changes
deleted file mode 100755
index 38e1f363dd9..00000000000
--- a/tooling/bin/find_changes
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'gitlab'
-
-class FindChanges # rubocop:disable Gitlab/NamespacedClass
- def initialize(output_file:, matched_tests_file: nil, frontend_fixtures_mapping_path: nil)
- @gitlab_token = ENV.fetch('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE', '')
- @gitlab_endpoint = ENV.fetch('CI_API_V4_URL')
- @mr_project_path = ENV.fetch('CI_MERGE_REQUEST_PROJECT_PATH')
- @mr_iid = ENV.fetch('CI_MERGE_REQUEST_IID')
- @output_file = output_file
- @matched_tests_file = matched_tests_file
- @frontend_fixtures_mapping_path = frontend_fixtures_mapping_path
- end
-
- def execute
- add_frontend_fixture_files!
-
- File.write(output_file, file_changes.join(' '))
- end
-
- private
-
- def add_frontend_fixture_files?
- matched_tests_file && frontend_fixtures_mapping_path
- end
-
- def add_frontend_fixture_files!
- return unless add_frontend_fixture_files?
-
- # If we have a `test file -> JSON frontend fixture` mapping file, we add the files JSON frontend fixtures
- # files to the list of changed files so that Jest can automatically run the dependent tests thanks to --findRelatedTests
- test_files.each do |test_file|
- file_changes.concat(frontend_fixtures_mapping[test_file]) if frontend_fixtures_mapping.key?(test_file)
- end
- end
-
- def file_changes
- @file_changes ||=
- if File.exist?(output_file)
- File.read(output_file).split(' ')
- else
- Gitlab.configure do |config|
- config.endpoint = gitlab_endpoint
- config.private_token = gitlab_token
- end
-
- mr_changes.changes.flat_map do |change|
- change.to_h.values_at('old_path', 'new_path')
- end.uniq
- end
- end
-
- def mr_changes
- @mr_changes ||= Gitlab.merge_request_changes(mr_project_path, mr_iid)
- end
-
- def test_files
- return [] if !matched_tests_file || !File.exist?(matched_tests_file)
-
- File.read(matched_tests_file).split(' ')
- end
-
- def frontend_fixtures_mapping
- return {} if !frontend_fixtures_mapping_path || !File.exist?(frontend_fixtures_mapping_path)
-
- JSON.parse(File.read(frontend_fixtures_mapping_path)) # rubocop:disable Gitlab/Json
- end
-
- attr_reader :gitlab_token, :gitlab_endpoint, :mr_project_path, :mr_iid, :output_file, :matched_tests_file, :frontend_fixtures_mapping_path
-end
-
-output_file = ARGV.shift
-raise ArgumentError, "An path to an output file must be given as first argument of #{__FILE__}." if output_file.nil?
-
-matched_tests_file = ARGV.shift
-frontend_fixtures_mapping_path = ARGV.shift
-
-FindChanges
- .new(output_file: output_file, matched_tests_file: matched_tests_file, frontend_fixtures_mapping_path: frontend_fixtures_mapping_path)
- .execute
diff --git a/tooling/bin/find_only_js_changes b/tooling/bin/find_only_js_changes
new file mode 100755
index 00000000000..a69ee64fe14
--- /dev/null
+++ b/tooling/bin/find_only_js_changes
@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/tooling/find_changes'
+
+if Tooling::FindChanges.new(from: :api).only_js_files_changed
+ puts "Only JS files were changed"
+ exit 0
+else
+ puts "Changes were made to files other than JS files"
+ exit 1
+end
diff --git a/tooling/bin/find_tests b/tooling/bin/find_tests
deleted file mode 100755
index 33834064f36..00000000000
--- a/tooling/bin/find_tests
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'test_file_finder'
-
-changes = ARGV.shift
-output_file = ARGV.shift
-
-changed_files = File.read(changes).split(' ')
-
-tff = TestFileFinder::FileFinder.new(paths: changed_files).tap do |file_finder|
- file_finder.use TestFileFinder::MappingStrategies::PatternMatching.load('tests.yml')
-
- if ENV['RSPEC_TESTS_MAPPING_ENABLED']
- file_finder.use TestFileFinder::MappingStrategies::DirectMatching.load_json(ENV['RSPEC_TESTS_MAPPING_PATH'])
- end
-end
-
-File.write(output_file, tff.test_files.uniq.join(' '))
diff --git a/tooling/bin/gettext_extractor b/tooling/bin/gettext_extractor
new file mode 100755
index 00000000000..39f029616df
--- /dev/null
+++ b/tooling/bin/gettext_extractor
@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/tooling/gettext_extractor'
+
+pot_file = ARGV.shift
+
+if !pot_file || !Dir.exist?(File.dirname(pot_file))
+ abort <<~MSG
+ Please provide a target file name as the first argument, e.g.
+ #{$PROGRAM_NAME} locale/gitlab.pot
+ MSG
+end
+
+puts <<~MSG
+ Extracting translatable strings from source files...
+MSG
+
+root_dir = File.expand_path('../../', __dir__)
+
+extractor = Tooling::GettextExtractor.new(
+ glob_base: root_dir
+)
+
+File.write(pot_file, extractor.generate_pot)
+
+puts <<~MSG
+ All done. Please commit the changes to `#{pot_file}`.
+MSG
diff --git a/tooling/bin/js_to_system_specs_mappings b/tooling/bin/js_to_system_specs_mappings
deleted file mode 100755
index 3e9d9cb4c5f..00000000000
--- a/tooling/bin/js_to_system_specs_mappings
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require_relative '../lib/tooling/mappings/js_to_system_specs_mappings'
-
-changes = ARGV.shift
-matching_tests = ARGV.shift
-
-changed_files = File.read(changes).split(' ')
-matching_test_files = File.read(matching_tests).split(' ')
-
-system_tests = Tooling::Mappings::JsToSystemSpecsMappings.new.execute(changed_files)
-
-File.write(matching_tests, (matching_test_files + system_tests).join(' '))
diff --git a/tooling/bin/predictive_tests b/tooling/bin/predictive_tests
new file mode 100755
index 00000000000..61543494a69
--- /dev/null
+++ b/tooling/bin/predictive_tests
@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/tooling/predictive_tests'
+
+Tooling::PredictiveTests.new.execute
diff --git a/tooling/bin/view_to_js_mappings b/tooling/bin/view_to_js_mappings
deleted file mode 100755
index 483003aac5e..00000000000
--- a/tooling/bin/view_to_js_mappings
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require_relative '../lib/tooling/mappings/view_to_js_mappings'
-
-changes = ARGV.shift
-output_file = ARGV.shift
-changed_files = File.read(changes).split(' ')
-
-File.write(output_file, Tooling::Mappings::ViewToJsMappings.new.execute(changed_files).join(' '))
diff --git a/tooling/danger/feature_flag.rb b/tooling/danger/feature_flag.rb
index da0b7053af1..3fb20c561af 100644
--- a/tooling/danger/feature_flag.rb
+++ b/tooling/danger/feature_flag.rb
@@ -15,6 +15,12 @@ module Tooling
files.select { |path| path =~ %r{\A(ee/)?config/feature_flags/} }.map { |path| Found.new(path) }
end
+ # TODO: Move this to gitlab-dangerfiles helper
+ # https://gitlab.com/gitlab-org/ruby/gems/gitlab-dangerfiles/-/blob/master/lib/danger/plugins/internal/helper.rb
+ def stage_label
+ helper.mr_labels.find { |label| label.start_with?("devops::") }
+ end
+
class Found
ATTRIBUTES = %w[name introduced_by_url rollout_issue_url milestone type group default_enabled].freeze
diff --git a/tooling/danger/multiversion.rb b/tooling/danger/multiversion.rb
new file mode 100644
index 00000000000..612881aaa1a
--- /dev/null
+++ b/tooling/danger/multiversion.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Tooling
+ module Danger
+ module Multiversion
+ FRONTEND_REGEX = %r{\A((ee|jh)/)?app/assets/.*(\.(vue|js|graphql))\z}
+ GRAPHQL_BACKEND_REGEX = %r{\A((ee|jh)/)?app/graphql/}
+
+ def check!
+ return unless helper.ci?
+ return unless frontend_changed? && backend_changed?
+
+ markdown <<~MARKDOWN
+ ## Multiversion compatibility
+
+ This merge request updates GraphQL backend and frontend code.
+
+ To prevent an incident, ensure the updated frontend code is backwards compatible.
+
+ For more information, see the [multiversion compatibility documentation](https://docs.gitlab.com/ee/development/graphql_guide/reviewing.html#multiversion-compatibility).
+ MARKDOWN
+ end
+
+ private
+
+ def frontend_changed?
+ !git.modified_files.grep(FRONTEND_REGEX).empty? || !git.added_files.grep(FRONTEND_REGEX).empty?
+ end
+
+ def backend_changed?
+ !git.added_files.grep(GRAPHQL_BACKEND_REGEX).empty? || !git.modified_files.grep(GRAPHQL_BACKEND_REGEX).empty?
+ end
+ end
+ end
+end
diff --git a/tooling/danger/specs.rb b/tooling/danger/specs.rb
index 5359e71f8cc..68cab141656 100644
--- a/tooling/danger/specs.rb
+++ b/tooling/danger/specs.rb
@@ -1,55 +1,19 @@
# frozen_string_literal: true
-require_relative 'suggestor'
+Dir[File.expand_path('specs/*_suggestion.rb', __dir__)].each { |file| require file }
module Tooling
module Danger
module Specs
- include ::Tooling::Danger::Suggestor
-
SPEC_FILES_REGEX = 'spec/'
EE_PREFIX = 'ee/'
- PROJECT_FACTORIES = %w[
- :project
- :project_empty_repo
- :forked_project_with_submodules
- :project_with_design
+ SUGGESTIONS = [
+ FeatureCategorySuggestion,
+ MatchWithArraySuggestion,
+ ProjectFactorySuggestion
].freeze
- PROJECT_FACTORY_REGEX = /
- ^\+? # Start of the line, which may or may not have a `+`
- (?<head>\s*) # 0-many leading whitespace captured in a group named head
- let!? # Literal `let` which may or may not end in `!`
- (?<tail> # capture group named tail
- \([^)]+\) # Two parenthesis with any non-parenthesis characters between them
- \s*\{\s* # Opening curly brace surrounded by 0-many whitespace characters
- create\( # literal
- (?:#{PROJECT_FACTORIES.join('|')}) # Any of the project factory names
- \W # Non-word character, avoid matching factories like :project_authorization
- ) # end capture group named tail
- /x.freeze
-
- PROJECT_FACTORY_REPLACEMENT = '\k<head>let_it_be\k<tail>'
- PROJECT_FACTORY_SUGGESTION = <<~SUGGEST_COMMENT
- Project creations are very slow. Use `let_it_be`, `build` or `build_stubbed` if possible.
- See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#optimize-factory-usage)
- for background information and alternative options.
- SUGGEST_COMMENT
-
- MATCH_WITH_ARRAY_REGEX = /(?<to>to\(?\s*)(?<matcher>match|eq)(?<expectation>[( ]?\[(?=.*,)[^\]]+)/.freeze
- MATCH_WITH_ARRAY_REPLACEMENT = '\k<to>match_array\k<expectation>'
- MATCH_WITH_ARRAY_SUGGESTION = <<~SUGGEST_COMMENT
- If order of the result is not important, please consider using `match_array` to avoid flakiness.
- SUGGEST_COMMENT
-
- RSPEC_TOP_LEVEL_DESCRIBE_REGEX = /^\+.?RSpec\.describe(.+)/.freeze
- FEATURE_CATEGORY_SUGGESTION = <<~SUGGESTION_MARKDOWN
- Consider adding `feature_category: <feature_category_name>` for this example if it is not set already.
- See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#feature-category-metadata).
- SUGGESTION_MARKDOWN
- FEATURE_CATEGORY_KEYWORD = 'feature_category'
-
def changed_specs_files(ee: :include)
changed_files = helper.all_changed_files
folder_prefix =
@@ -65,45 +29,9 @@ module Tooling
changed_files.grep(%r{\A#{folder_prefix}#{SPEC_FILES_REGEX}})
end
- def add_suggestions_for_match_with_array(filename)
- add_suggestion(
- filename: filename,
- regex: MATCH_WITH_ARRAY_REGEX,
- replacement: MATCH_WITH_ARRAY_REPLACEMENT,
- comment_text: MATCH_WITH_ARRAY_SUGGESTION
- )
- end
-
- def add_suggestions_for_project_factory_usage(filename)
- add_suggestion(
- filename: filename,
- regex: PROJECT_FACTORY_REGEX,
- replacement: PROJECT_FACTORY_REPLACEMENT,
- comment_text: PROJECT_FACTORY_SUGGESTION
- )
- end
-
- def add_suggestions_for_feature_category(filename)
- file_lines = project_helper.file_lines(filename)
- changed_lines = helper.changed_lines(filename)
-
- changed_lines.each_with_index do |changed_line, i|
- next unless changed_line =~ RSPEC_TOP_LEVEL_DESCRIBE_REGEX
-
- line_number = file_lines.find_index(changed_line.delete_prefix('+'))
- next unless line_number
-
- # Get the top level RSpec.describe line and the next 5 lines
- lines_to_check = file_lines[line_number, 5]
- # Remove all the lines after the first one that ends in `do`
- last_line_number_of_describe_declaration = lines_to_check.index { |line| line.end_with?(' do') }
- lines_to_check = lines_to_check[0..last_line_number_of_describe_declaration]
-
- next if lines_to_check.any? { |line| line.include?(FEATURE_CATEGORY_KEYWORD) }
-
- suggested_line = file_lines[line_number]
-
- markdown(comment(FEATURE_CATEGORY_SUGGESTION, suggested_line), file: filename, line: line_number.succ)
+ def add_suggestions_for(filename)
+ SUGGESTIONS.each do |suggestion|
+ suggestion.new(filename, context: self).suggest
end
end
end
diff --git a/tooling/danger/specs/feature_category_suggestion.rb b/tooling/danger/specs/feature_category_suggestion.rb
new file mode 100644
index 00000000000..5acf73c8956
--- /dev/null
+++ b/tooling/danger/specs/feature_category_suggestion.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative '../suggestion'
+
+module Tooling
+ module Danger
+ module Specs
+ class FeatureCategorySuggestion < Suggestion
+ RSPEC_TOP_LEVEL_DESCRIBE_REGEX = /^\+.?RSpec\.describe(.+)/
+ SUGGESTION = <<~SUGGESTION_MARKDOWN
+ Consider adding `feature_category: <feature_category_name>` for this example if it is not set already.
+ See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#feature-category-metadata).
+ SUGGESTION_MARKDOWN
+ FEATURE_CATEGORY_KEYWORD = 'feature_category'
+
+ def suggest
+ file_lines = project_helper.file_lines(filename)
+ changed_lines = helper.changed_lines(filename)
+
+ changed_lines.each do |changed_line|
+ next unless changed_line =~ RSPEC_TOP_LEVEL_DESCRIBE_REGEX
+
+ line_number = file_lines.find_index(changed_line.delete_prefix('+'))
+ next unless line_number
+
+ # Get the top level RSpec.describe line and the next 5 lines
+ lines_to_check = file_lines[line_number, 5]
+ # Remove all the lines after the first one that ends in `do`
+ last_line_number_of_describe_declaration = lines_to_check.index { |line| line.end_with?(' do') }
+ lines_to_check = lines_to_check[0..last_line_number_of_describe_declaration]
+
+ next if lines_to_check.any? { |line| line.include?(FEATURE_CATEGORY_KEYWORD) }
+
+ suggested_line = file_lines[line_number]
+
+ markdown(comment(SUGGESTION, suggested_line), file: filename, line: line_number.succ)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/tooling/danger/specs/match_with_array_suggestion.rb b/tooling/danger/specs/match_with_array_suggestion.rb
new file mode 100644
index 00000000000..eb0f7a1a832
--- /dev/null
+++ b/tooling/danger/specs/match_with_array_suggestion.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative '../suggestion'
+
+module Tooling
+ module Danger
+ module Specs
+ class MatchWithArraySuggestion < Suggestion
+ MATCH = /(?<to>to\(?\s*)(?<matcher>match|eq)(?<expectation>[( ]?\[(?=.*,)[^\]]+)/
+ REPLACEMENT = '\k<to>match_array\k<expectation>'
+ SUGGESTION = <<~SUGGEST_COMMENT
+ If order of the result is not important, please consider using `match_array` to avoid flakiness.
+ SUGGEST_COMMENT
+ end
+ end
+ end
+end
diff --git a/tooling/danger/specs/project_factory_suggestion.rb b/tooling/danger/specs/project_factory_suggestion.rb
new file mode 100644
index 00000000000..4e5a70ac8e5
--- /dev/null
+++ b/tooling/danger/specs/project_factory_suggestion.rb
@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative '../suggestion'
+
+module Tooling
+ module Danger
+ module Specs
+ class ProjectFactorySuggestion < Suggestion
+ PROJECT_FACTORIES = %w[
+ :project
+ :project_empty_repo
+ :forked_project_with_submodules
+ :project_with_design
+ ].freeze
+
+ MATCH = /
+ ^\+? # Start of the line, which may or may not have a `+`
+ (?<head>\s*) # 0-many leading whitespace captured in a group named head
+ let!? # Literal `let` which may or may not end in `!`
+ (?<tail> # capture group named tail
+ \([^)]+\) # Two parenthesis with any non-parenthesis characters between them
+ \s*\{\s* # Opening curly brace surrounded by 0-many whitespace characters
+ create\( # literal
+ (?:#{PROJECT_FACTORIES.join('|')}) # Any of the project factory names
+ \W # Non-word character, avoid matching factories like :project_badge
+ ) # end capture group named tail
+ /x
+
+ REPLACEMENT = '\k<head>let_it_be\k<tail>'
+ SUGGESTION = <<~SUGGEST_COMMENT
+ Project creations are very slow. Use `let_it_be`, `build` or `build_stubbed` if possible.
+ See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#optimize-factory-usage)
+ for background information and alternative options.
+ SUGGEST_COMMENT
+ end
+ end
+ end
+end
diff --git a/tooling/danger/stable_branch.rb b/tooling/danger/stable_branch.rb
index 2751a6f0191..9deb4838079 100644
--- a/tooling/danger/stable_branch.rb
+++ b/tooling/danger/stable_branch.rb
@@ -85,12 +85,12 @@ module Tooling
!has_flaky_failure_label?
end
- private
-
def valid_stable_branch?
!!stable_target_branch && !helper.security_mr?
end
+ private
+
def package_and_test_bridge_and_pipeline_status
mr_head_pipeline_id = gitlab.mr_json.dig('head_pipeline', 'id')
return unless mr_head_pipeline_id
diff --git a/tooling/danger/suggestion.rb b/tooling/danger/suggestion.rb
new file mode 100644
index 00000000000..da3c6b0e76f
--- /dev/null
+++ b/tooling/danger/suggestion.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require_relative 'suggestor'
+
+module Tooling
+ module Danger
+ # A basic suggestion.
+ #
+ # A subclass needs to define the following constants:
+ # * MATCH (Regexp) - A Regexp to match file lines
+ # * REPLACEMENT (String) - A suggestion replacement text
+ # * SUGGESTION (String) - A suggestion text
+ #
+ # @see Suggestor
+ class Suggestion
+ extend Forwardable
+ include ::Tooling::Danger::Suggestor
+
+ def_delegators :@context, :helper, :project_helper, :markdown
+
+ attr_reader :filename
+
+ def initialize(filename, context:)
+ @filename = filename
+ @context = context
+ end
+
+ def suggest
+ add_suggestion(
+ filename: filename,
+ regex: self.class::MATCH,
+ replacement: self.class::REPLACEMENT,
+ comment_text: self.class::SUGGESTION
+ )
+ end
+ end
+ end
+end
diff --git a/tooling/docs/deprecation_handling.rb b/tooling/docs/deprecation_handling.rb
index bcdf73e0044..5996a0c89c1 100644
--- a/tooling/docs/deprecation_handling.rb
+++ b/tooling/docs/deprecation_handling.rb
@@ -1,4 +1,5 @@
# frozen_string_literal: true
+
require 'erb'
module Docs
@@ -7,11 +8,7 @@ module Docs
@type = type
@yaml_glob_path = Rails.root.join("data/#{type.pluralize}/**/*.yml")
@template_path = Rails.root.join("data/#{type.pluralize}/templates/_#{type}_template.md.erb")
- @milestone_key_name = if type == "deprecation"
- "announcement_milestone"
- else
- "removal_milestone"
- end
+ @milestone_key_name = "removal_milestone"
end
def render
diff --git a/tooling/lib/tooling/find_changes.rb b/tooling/lib/tooling/find_changes.rb
new file mode 100755
index 00000000000..d8373d11245
--- /dev/null
+++ b/tooling/lib/tooling/find_changes.rb
@@ -0,0 +1,107 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'gitlab'
+require_relative 'helpers/file_handler'
+
+module Tooling
+ class FindChanges
+ include Helpers::FileHandler
+
+ def initialize(
+ from:,
+ changed_files_pathname: nil,
+ predictive_tests_pathname: nil,
+ frontend_fixtures_mapping_pathname: nil
+ )
+
+ raise ArgumentError, ':from can only be :api or :changed_files' unless
+ %i[api changed_files].include?(from)
+
+ @gitlab_token = ENV['PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE'] || ''
+ @gitlab_endpoint = ENV['CI_API_V4_URL']
+ @mr_project_path = ENV['CI_MERGE_REQUEST_PROJECT_PATH']
+ @mr_iid = ENV['CI_MERGE_REQUEST_IID']
+ @changed_files_pathname = changed_files_pathname
+ @predictive_tests_pathname = predictive_tests_pathname
+ @frontend_fixtures_mapping_pathname = frontend_fixtures_mapping_pathname
+ @from = from
+ end
+
+ def execute
+ if changed_files_pathname.nil?
+ raise ArgumentError, "A path to the changed files file must be given as :changed_files_pathname"
+ end
+
+ case @from
+ when :api
+ write_array_to_file(changed_files_pathname, file_changes + frontend_fixture_files, append: false)
+ else
+ write_array_to_file(changed_files_pathname, frontend_fixture_files, append: true)
+ end
+ end
+
+ def only_js_files_changed
+ file_changes.any? && file_changes.all? { |file| file.end_with?('.js') }
+ end
+
+ private
+
+ attr_reader :gitlab_token, :gitlab_endpoint, :mr_project_path,
+ :mr_iid, :changed_files_pathname, :predictive_tests_pathname, :frontend_fixtures_mapping_pathname
+
+ def gitlab
+ @gitlab ||= begin
+ Gitlab.configure do |config|
+ config.endpoint = gitlab_endpoint
+ config.private_token = gitlab_token
+ end
+
+ Gitlab
+ end
+ end
+
+ def add_frontend_fixture_files?
+ predictive_tests_pathname && frontend_fixtures_mapping_pathname
+ end
+
+ def frontend_fixture_files
+ # If we have a `test file -> JSON frontend fixture` mapping file, we add the files JSON frontend fixtures
+ # files to the list of changed files so that Jest can automatically run the dependent tests
+ # using --findRelatedTests flag.
+ empty = [].freeze
+
+ test_files.flat_map do |test_file|
+ frontend_fixtures_mapping[test_file] || empty
+ end
+ end
+
+ def file_changes
+ @file_changes ||=
+ case @from
+ when :api
+ mr_changes.changes.flat_map do |change|
+ change.to_h.values_at('old_path', 'new_path')
+ end.uniq
+ else
+ read_array_from_file(changed_files_pathname)
+ end
+ end
+
+ def mr_changes
+ @mr_changes ||= gitlab.merge_request_changes(mr_project_path, mr_iid)
+ end
+
+ def test_files
+ return [] if !predictive_tests_pathname || !File.exist?(predictive_tests_pathname)
+
+ read_array_from_file(predictive_tests_pathname)
+ end
+
+ def frontend_fixtures_mapping
+ return {} if !frontend_fixtures_mapping_pathname || !File.exist?(frontend_fixtures_mapping_pathname)
+
+ JSON.parse(File.read(frontend_fixtures_mapping_pathname)) # rubocop:disable Gitlab/Json
+ end
+ end
+end
diff --git a/tooling/lib/tooling/find_tests.rb b/tooling/lib/tooling/find_tests.rb
new file mode 100644
index 00000000000..f26c1eacdc7
--- /dev/null
+++ b/tooling/lib/tooling/find_tests.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'test_file_finder'
+require_relative 'helpers/file_handler'
+
+module Tooling
+ class FindTests
+ include Helpers::FileHandler
+
+ def initialize(changed_files_pathname, predictive_tests_pathname)
+ @predictive_tests_pathname = predictive_tests_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ end
+
+ def execute
+ tff = TestFileFinder::FileFinder.new(paths: changed_files).tap do |file_finder|
+ file_finder.use TestFileFinder::MappingStrategies::PatternMatching.load('tests.yml')
+
+ if ENV['RSPEC_TESTS_MAPPING_ENABLED'] == 'true'
+ file_finder.use TestFileFinder::MappingStrategies::DirectMatching.load_json(ENV['RSPEC_TESTS_MAPPING_PATH'])
+ end
+ end
+
+ write_array_to_file(predictive_tests_pathname, tff.test_files.uniq)
+ end
+
+ private
+
+ attr_reader :changed_files, :matching_tests, :predictive_tests_pathname
+ end
+end
diff --git a/tooling/lib/tooling/gettext_extractor.rb b/tooling/lib/tooling/gettext_extractor.rb
new file mode 100644
index 00000000000..673749a5a16
--- /dev/null
+++ b/tooling/lib/tooling/gettext_extractor.rb
@@ -0,0 +1,106 @@
+# frozen_string_literal: true
+
+require 'parallel'
+require 'gettext/po'
+require 'gettext/po_entry'
+require 'gettext/tools/parser/erb'
+require 'gettext/tools/parser/ruby'
+require 'gettext_i18n_rails/haml_parser'
+require 'json'
+require 'open3'
+
+module Tooling
+ class GettextExtractor < GetText::Tools::XGetText
+ class HamlParser < GettextI18nRails::HamlParser
+ # If both `haml` and `hamlit` are available,
+ # the parser prefers `haml`. `hamlit` should be faster
+ def self.libraries
+ ["hamlit"]
+ end
+ end
+
+ def initialize(
+ backend_glob: "{ee,app,lib,config,locale}/**/*.{rb,erb,haml}",
+ glob_base: nil,
+ package_name: 'gitlab',
+ package_version: '1.0.0'
+ )
+ super()
+ @backend_glob = backend_glob
+ @package_name = package_name
+ @glob_base = glob_base || Dir.pwd
+ @package_version = package_version
+ # Ensure that the messages are ordered by id
+ @po_order = :msgid
+ @po_format_options = {
+ # No line breaks within a message
+ max_line_width: -1,
+ # Do not print references to files
+ include_reference_comment: false
+ }
+ end
+
+ def parse(_paths)
+ po = GetText::PO.new
+ parse_backend_files.each do |po_entry|
+ merge_po_entries(po, po_entry)
+ end
+ parse_frontend_files.each do |po_entry|
+ merge_po_entries(po, po_entry)
+ end
+ po
+ end
+
+ # Overrides method from GetText::Tools::XGetText
+ # This makes a method public and passes in an empty array of paths,
+ # as our overidden "parse" method needs no paths
+ def generate_pot
+ super([])
+ end
+
+ private
+
+ # Overrides method from GetText::Tools::XGetText
+ # in order to remove revision dates, as we check in our locale/gitlab.pot
+ def header_content
+ super.gsub(/^POT?-(?:Creation|Revision)-Date:.*\n/, '')
+ end
+
+ def merge_po_entries(po, po_entry)
+ existing_entry = po[po_entry.msgctxt, po_entry.msgid]
+ po_entry = existing_entry.merge(po_entry) if existing_entry
+
+ po[po_entry.msgctxt, po_entry.msgid] = po_entry
+ end
+
+ def parse_backend_file(path)
+ case ::File.extname(path)
+ when '.rb'
+ GetText::RubyParser.new(path).parse
+ when '.haml'
+ HamlParser.parse(path).collect { |item| create_po_entry(*item) }
+ when '.erb'
+ GetText::ErbParser.new(path).parse
+ else
+ raise NotImplementedError
+ end
+ end
+
+ def parse_backend_files
+ files = Dir.glob(File.join(@glob_base, @backend_glob))
+ Parallel.flat_map(files) { |item| parse_backend_file(item) }
+ end
+
+ def parse_frontend_files
+ results, status = Open3.capture2('node scripts/frontend/extract_gettext_all.js --all')
+ raise StandardError, "Could not parse frontend files" unless status.success?
+
+ # rubocop:disable Gitlab/Json
+ JSON.parse(results)
+ .values
+ .flatten(1)
+ .collect { |entry| create_po_entry(*entry) }
+ # rubocop:enable Gitlab/Json
+ end
+ end
+end
diff --git a/tooling/lib/tooling/helpers/file_handler.rb b/tooling/lib/tooling/helpers/file_handler.rb
new file mode 100644
index 00000000000..88248e31df2
--- /dev/null
+++ b/tooling/lib/tooling/helpers/file_handler.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module Tooling
+ module Helpers
+ module FileHandler
+ def read_array_from_file(file)
+ FileUtils.touch file
+
+ File.read(file).split(' ')
+ end
+
+ def write_array_to_file(file, content_array, append: true)
+ FileUtils.touch file
+
+ # We sort the array to make it easier to read the output file
+ content_array.sort!
+
+ output_content =
+ if append
+ [File.read(file), *content_array].join(' ').lstrip
+ else
+ content_array.join(' ')
+ end
+
+ File.write(file, output_content)
+ end
+ end
+ end
+end
diff --git a/tooling/lib/tooling/kubernetes_client.rb b/tooling/lib/tooling/kubernetes_client.rb
index 27eb4c8151e..5579f130a84 100644
--- a/tooling/lib/tooling/kubernetes_client.rb
+++ b/tooling/lib/tooling/kubernetes_client.rb
@@ -6,33 +6,23 @@ require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen)
module Tooling
class KubernetesClient
- RESOURCE_LIST = 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd'
K8S_ALLOWED_NAMESPACES_REGEX = /^review-(?!apps).+/.freeze
CommandFailedError = Class.new(StandardError)
- attr_reader :namespace
+ def cleanup_namespaces_by_created_at(created_before:)
+ stale_namespaces = namespaces_created_before(created_before: created_before)
- def initialize(namespace:)
- @namespace = namespace
- end
-
- def cleanup_by_release(release_name:, wait: true)
- delete_by_selector(release_name: release_name, wait: wait)
- delete_by_matching_name(release_name: release_name)
- end
-
- def cleanup_by_created_at(resource_type:, created_before:, wait: true)
- resource_names = resource_names_created_before(resource_type: resource_type, created_before: created_before)
- return if resource_names.empty?
+ # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby.
+ review_apps_stale_namespaces = stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) }
+ return if review_apps_stale_namespaces.empty?
- delete_by_exact_names(resource_type: resource_type, resource_names: resource_names, wait: wait)
+ delete_namespaces(review_apps_stale_namespaces)
end
- def cleanup_review_app_namespaces(created_before:, wait: true)
- namespaces = review_app_namespaces_created_before(created_before: created_before)
- return if namespaces.empty?
+ def delete_namespaces(namespaces)
+ return if namespaces.any? { |ns| !K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) }
- delete_namespaces_by_exact_names(resource_names: namespaces, wait: wait)
+ run_command("kubectl delete namespace --now --ignore-not-found #{namespaces.join(' ')}")
end
def delete_namespaces_by_exact_names(resource_names:, wait:)
@@ -48,126 +38,29 @@ module Tooling
run_command(command)
end
- private
-
- def delete_by_selector(release_name:, wait:)
- selector = case release_name
- when String
- %(-l release="#{release_name}")
- when Array
- %(-l 'release in (#{release_name.join(', ')})')
- else
- raise ArgumentError, 'release_name must be a string or an array'
- end
-
- command = [
- 'delete',
- RESOURCE_LIST,
- %(--namespace "#{namespace}"),
- '--now',
- '--ignore-not-found',
- %(--wait=#{wait}),
- selector
- ]
-
- run_command(command)
- end
-
- def delete_by_exact_names(resource_names:, wait:, resource_type: nil)
- command = [
- 'delete',
- resource_type,
- %(--namespace "#{namespace}"),
- '--now',
- '--ignore-not-found',
- %(--wait=#{wait}),
- resource_names.join(' ')
- ]
-
- run_command(command)
- end
-
- def delete_by_matching_name(release_name:)
- resource_names = raw_resource_names
- command = [
- 'delete',
- %(--namespace "#{namespace}"),
- '--ignore-not-found'
- ]
-
- Array(release_name).each do |release|
- resource_names
- .select { |resource_name| resource_name.include?(release) }
- .each { |matching_resource| run_command(command + [matching_resource]) }
- end
- end
-
- def raw_resource_names
- command = [
- 'get',
- RESOURCE_LIST,
- %(--namespace "#{namespace}"),
- '-o name'
- ]
- run_command(command).lines.map(&:strip)
- end
-
- def resource_names_created_before(resource_type:, created_before:)
- command = [
- 'get',
- resource_type,
- %(--namespace "#{namespace}"),
- "--sort-by='{.metadata.creationTimestamp}'",
- '-o json'
- ]
+ def namespaces_created_before(created_before:)
+ response = run_command("kubectl get namespace --all-namespaces --sort-by='{.metadata.creationTimestamp}' -o json")
- response = run_command(command)
-
- resources_created_before_date(response, created_before)
- end
-
- def review_app_namespaces_created_before(created_before:)
- command = [
- 'get',
- 'namespace',
- "--sort-by='{.metadata.creationTimestamp}'",
- '-o json'
- ]
-
- response = run_command(command)
-
- stale_namespaces = resources_created_before_date(response, created_before)
-
- # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby.
- stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) }
- end
-
- def resources_created_before_date(response, date)
items = JSON.parse(response)['items'] # rubocop:disable Gitlab/Json
-
- items.each_with_object([]) do |item, result|
+ items.filter_map do |item|
item_created_at = Time.parse(item.dig('metadata', 'creationTimestamp'))
- if item_created_at < date
- resource_name = item.dig('metadata', 'name')
- result << resource_name
- end
+ item.dig('metadata', 'name') if item_created_at < created_before
end
rescue ::JSON::ParserError => ex
- puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output
+ puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}"
[]
end
def run_command(command)
- final_command = ['kubectl', *command.compact].join(' ')
- puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
+ puts "Running command: `#{command}`"
- result = Gitlab::Popen.popen_with_detail([final_command])
+ result = Gitlab::Popen.popen_with_detail([command])
if result.status.success?
result.stdout.chomp.freeze
else
- raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
+ raise CommandFailedError, "The `#{command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
end
end
end
diff --git a/tooling/lib/tooling/mappings/base.rb b/tooling/lib/tooling/mappings/base.rb
index 93d3a967114..27a9a0925b0 100644
--- a/tooling/lib/tooling/mappings/base.rb
+++ b/tooling/lib/tooling/mappings/base.rb
@@ -1,22 +1,13 @@
# frozen_string_literal: true
require_relative '../../../../lib/gitlab_edition'
+require_relative '../helpers/file_handler'
# Returns system specs files that are related to the JS files that were changed in the MR.
module Tooling
module Mappings
class Base
- # Input: A list of space-separated files
- # Output: A list of space-separated specs files (JS, Ruby, ...)
- def execute(changed_files)
- raise "Not Implemented"
- end
-
- # Input: A list of space-separated files
- # Output: array/hash of files
- def filter_files(changed_files)
- raise "Not Implemented"
- end
+ include Helpers::FileHandler
# Input: A folder
# Output: An array of folders, each prefixed with a GitLab edition
diff --git a/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb b/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb
new file mode 100644
index 00000000000..569a8278163
--- /dev/null
+++ b/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb
@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'active_support/inflector'
+
+require_relative 'base'
+require_relative '../../../../lib/gitlab_edition'
+
+# If a GraphQL type class changed, we try to identify the other GraphQL types that potentially include this type.
+module Tooling
+ module Mappings
+ class GraphqlBaseTypeMappings < Base
+ # Checks for the implements keyword, and graphql_base_types the class name
+ GRAPHQL_IMPLEMENTS_REGEXP = /implements[( ]([\w:]+)[)]?$/
+
+ # GraphQL types are a bit scattered in the codebase based on the edition.
+ #
+ # Also, a higher edition is able to include lower editions.
+ # e.g. EE can include FOSS GraphQL types, and JH can include all GraphQL types
+ GRAPHQL_TYPES_FOLDERS_FOSS = ['app/graphql/types'].freeze
+ GRAPHQL_TYPES_FOLDERS_EE = GRAPHQL_TYPES_FOLDERS_FOSS + ['ee/app/graphql/types', 'ee/app/graphql/ee/types']
+ GRAPHQL_TYPES_FOLDERS_JH = GRAPHQL_TYPES_FOLDERS_EE + ['jh/app/graphql/types', 'jh/app/graphql/jh/types']
+ GRAPHQL_TYPES_FOLDERS = {
+ nil => GRAPHQL_TYPES_FOLDERS_FOSS,
+ 'ee' => GRAPHQL_TYPES_FOLDERS_EE,
+ 'jh' => GRAPHQL_TYPES_FOLDERS_JH
+ }.freeze
+
+ def initialize(changed_files_pathname, predictive_tests_pathname)
+ @predictive_tests_pathname = predictive_tests_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ end
+
+ def execute
+ # We go through the available editions when searching for base types
+ #
+ # `nil` is the FOSS edition
+ matching_graphql_tests = ([nil] + ::GitlabEdition.extensions).flat_map do |edition|
+ hierarchy = types_hierarchies[edition]
+
+ filter_files.flat_map do |graphql_file|
+ children_types = hierarchy[filename_to_class_name(graphql_file)]
+ next if children_types.empty?
+
+ # We find the specs for the children GraphQL types that are implementing the current GraphQL Type
+ children_types.map { |filename| filename_to_spec_filename(filename) }
+ end
+ end.compact.uniq
+
+ write_array_to_file(predictive_tests_pathname, matching_graphql_tests)
+ end
+
+ def filter_files
+ changed_files.select do |filename|
+ filename.start_with?(*GRAPHQL_TYPES_FOLDERS.values.flatten.uniq) &&
+ filename.end_with?('.rb') &&
+ File.exist?(filename)
+ end
+ end
+
+ # Regroup all GraphQL types (by edition) that are implementing another GraphQL type.
+ #
+ # The key is the type that is being implemented (e.g. NoteableInterface, TodoableInterface below)
+ # The value is an array of GraphQL type files that are implementing those types.
+ #
+ # Example output:
+ #
+ # {
+ # nil => {
+ # "NoteableInterface" => [
+ # "app/graphql/types/alert_management/alert_type.rb",
+ # "app/graphql/types/design_management/design_type.rb"
+ # , "TodoableInterface" => [...]
+ # },
+ # "ee" => {
+ # "NoteableInterface" => [
+ # "app/graphql/types/alert_management/alert_type.rb",
+ # "app/graphql/types/design_management/design_type.rb",
+ # "ee/app/graphql/types/epic_type.rb"],
+ # "TodoableInterface"=> [...]
+ # }
+ # }
+ def types_hierarchies
+ return @types_hierarchies if @types_hierarchies
+
+ @types_hierarchies = {}
+ GRAPHQL_TYPES_FOLDERS.each_key do |edition|
+ @types_hierarchies[edition] = Hash.new { |h, k| h[k] = [] }
+
+ graphql_files_for_edition_glob = File.join("{#{GRAPHQL_TYPES_FOLDERS[edition].join(',')}}", '**', '*.rb')
+ Dir[graphql_files_for_edition_glob].each do |graphql_file|
+ graphql_base_types = File.read(graphql_file).scan(GRAPHQL_IMPLEMENTS_REGEXP)
+ next if graphql_base_types.empty?
+
+ graphql_base_classes = graphql_base_types.flatten.map { |class_name| class_name.split('::').last }
+ graphql_base_classes.each do |graphql_base_class|
+ @types_hierarchies[edition][graphql_base_class] += [graphql_file]
+ end
+ end
+ end
+
+ @types_hierarchies
+ end
+
+ def filename_to_class_name(filename)
+ File.basename(filename, '.*').camelize
+ end
+
+ def filename_to_spec_filename(filename)
+ spec_file = filename.sub('app', 'spec').sub('.rb', '_spec.rb')
+
+ return spec_file if File.exist?(spec_file)
+ end
+
+ private
+
+ attr_reader :changed_files, :predictive_tests_pathname
+ end
+ end
+end
diff --git a/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb
index e78ce266a32..b2fca3a765a 100644
--- a/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb
+++ b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb
@@ -9,10 +9,14 @@ require_relative '../../../../lib/gitlab_edition'
module Tooling
module Mappings
class JsToSystemSpecsMappings < Base
- def initialize(js_base_folder: 'app/assets/javascripts', system_specs_base_folder: 'spec/features')
- @js_base_folder = js_base_folder
- @js_base_folders = folders_for_available_editions(js_base_folder)
- @system_specs_base_folder = system_specs_base_folder
+ def initialize(
+ changed_files_pathname, predictive_tests_pathname,
+ js_base_folder: 'app/assets/javascripts', system_specs_base_folder: 'spec/features')
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @predictive_tests_pathname = predictive_tests_pathname
+ @js_base_folder = js_base_folder
+ @js_base_folders = folders_for_available_editions(js_base_folder)
+ @system_specs_base_folder = system_specs_base_folder
# Cannot be extracted to a constant, as it depends on a variable
@first_js_folder_extract_regexp = %r{
@@ -23,20 +27,22 @@ module Tooling
}x
end
- def execute(changed_files)
- filter_files(changed_files).flat_map do |edition, js_files|
+ def execute
+ matching_system_tests = filter_files.flat_map do |edition, js_files|
js_keywords_regexp = Regexp.union(construct_js_keywords(js_files))
system_specs_for_edition(edition).select do |system_spec_file|
system_spec_file if js_keywords_regexp.match?(system_spec_file)
end
end
+
+ write_array_to_file(predictive_tests_pathname, matching_system_tests)
end
# Keep the files that are in the @js_base_folders folders
#
# Returns a hash, where the key is the GitLab edition, and the values the JS specs
- def filter_files(changed_files)
+ def filter_files
selected_files = changed_files.select do |filename|
filename.start_with?(*@js_base_folders) && File.exist?(filename)
end
@@ -55,8 +61,12 @@ module Tooling
def system_specs_for_edition(edition)
all_files_in_folders_glob = File.join(@system_specs_base_folder, '**', '*')
all_files_in_folders_glob = File.join(edition, all_files_in_folders_glob) if edition
- Dir[all_files_in_folders_glob].select { |f| File.file?(f) }
+ Dir[all_files_in_folders_glob].select { |f| File.file?(f) && f.end_with?('_spec.rb') }
end
+
+ private
+
+ attr_reader :changed_files, :predictive_tests_pathname
end
end
end
diff --git a/tooling/lib/tooling/mappings/partial_to_views_mappings.rb b/tooling/lib/tooling/mappings/partial_to_views_mappings.rb
new file mode 100644
index 00000000000..8b0a5ed4ecd
--- /dev/null
+++ b/tooling/lib/tooling/mappings/partial_to_views_mappings.rb
@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require_relative '../../../../lib/gitlab_edition'
+
+# Returns view files that include the potential rails partials from the changed files passed as input.
+module Tooling
+ module Mappings
+ class PartialToViewsMappings < Base
+ def initialize(changed_files_pathname, views_with_partials_pathname, view_base_folder: 'app/views')
+ @views_with_partials_pathname = views_with_partials_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @view_base_folders = folders_for_available_editions(view_base_folder)
+ end
+
+ def execute
+ views_including_modified_partials = []
+
+ views_globs = view_base_folders.map { |view_base_folder| "#{view_base_folder}/**/*.html.haml" }
+ Dir[*views_globs].each do |view_file|
+ included_partial_names = find_pattern_in_file(view_file, partials_keywords_regexp)
+ next if included_partial_names.empty?
+
+ included_partial_names.each do |included_partial_name|
+ if view_includes_modified_partial?(view_file, included_partial_name)
+ views_including_modified_partials << view_file
+ end
+ end
+ end
+
+ write_array_to_file(views_with_partials_pathname, views_including_modified_partials)
+ end
+
+ def filter_files
+ @_filter_files ||= changed_files.select do |filename|
+ filename.start_with?(*view_base_folders) &&
+ File.basename(filename).start_with?('_') &&
+ File.basename(filename).end_with?('.html.haml') &&
+ File.exist?(filename)
+ end
+ end
+
+ def partials_keywords_regexp
+ partial_keywords = filter_files.map do |partial_filename|
+ extract_partial_keyword(partial_filename)
+ end
+
+ partial_regexps = partial_keywords.map do |keyword|
+ %r{(?:render|render_if_exists)(?: |\()(?:partial: ?)?['"]([\w\-_/]*#{keyword})['"]}
+ end
+
+ Regexp.union(partial_regexps)
+ end
+
+ # e.g. if app/views/clusters/clusters/_sidebar.html.haml was modified, the partial keyword is `sidebar`.
+ def extract_partial_keyword(partial_filename)
+ File.basename(partial_filename).delete_prefix('_').delete_suffix('.html.haml')
+ end
+
+ # Why do we need this method?
+ #
+ # Assume app/views/clusters/clusters/_sidebar.html.haml was modified in the MR.
+ #
+ # Suppose now you find = render 'sidebar' in a view. Is this view including the sidebar partial
+ # that was modified, or another partial called "_sidebar.html.haml" somewhere else?
+ def view_includes_modified_partial?(view_file, included_partial_name)
+ view_file_parent_folder = File.dirname(view_file)
+ included_partial_filename = reconstruct_partial_filename(included_partial_name)
+ included_partial_relative_path = File.join(view_file_parent_folder, included_partial_filename)
+
+ # We do this because in render (or render_if_exists)
+ # apparently looks for partials in other GitLab editions
+ #
+ # Example:
+ #
+ # ee/app/views/events/_epics_filter.html.haml is used in app/views/shared/_event_filter.html.haml
+ # with render_if_exists 'events/epics_filter'
+ included_partial_absolute_paths = view_base_folders.map do |view_base_folder|
+ File.join(view_base_folder, included_partial_filename)
+ end
+
+ filter_files.include?(included_partial_relative_path) ||
+ (filter_files & included_partial_absolute_paths).any?
+ end
+
+ def reconstruct_partial_filename(partial_name)
+ partial_path = partial_name.split('/')[..-2]
+ partial_filename = partial_name.split('/').last
+ full_partial_filename = "_#{partial_filename}.html.haml"
+
+ return full_partial_filename if partial_path.empty?
+
+ File.join(partial_path.join('/'), full_partial_filename)
+ end
+
+ def find_pattern_in_file(file, pattern)
+ File.read(file).scan(pattern).flatten.compact.uniq
+ end
+
+ private
+
+ attr_reader :changed_files, :views_with_partials_pathname, :view_base_folders
+ end
+ end
+end
diff --git a/tooling/lib/tooling/mappings/view_to_js_mappings.rb b/tooling/lib/tooling/mappings/view_to_js_mappings.rb
index db80eb9bfe8..f2098d6acd5 100644
--- a/tooling/lib/tooling/mappings/view_to_js_mappings.rb
+++ b/tooling/lib/tooling/mappings/view_to_js_mappings.rb
@@ -11,37 +11,41 @@ module Tooling
HTML_ATTRIBUTE_VALUE_REGEXP = /js-[-\w]+/.freeze
# Search for Rails partials included in an HTML file
- RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exist)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze
+ RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exists)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze
- def initialize(view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts')
- @view_base_folders = folders_for_available_editions(view_base_folder)
- @js_base_folders = folders_for_available_editions(js_base_folder)
+ def initialize(
+ changed_files_pathname, predictive_tests_pathname,
+ view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts')
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @predictive_tests_pathname = predictive_tests_pathname
+ @view_base_folders = folders_for_available_editions(view_base_folder)
+ @js_base_folders = folders_for_available_editions(js_base_folder)
end
- def execute(changed_files)
- changed_view_files = filter_files(changed_files)
-
- partials = changed_view_files.flat_map do |file|
+ def execute
+ partials = filter_files.flat_map do |file|
find_partials(file)
end
- files_to_scan = changed_view_files + partials
+ files_to_scan = filter_files + partials
js_tags = files_to_scan.flat_map do |file|
find_pattern_in_file(file, HTML_ATTRIBUTE_VALUE_REGEXP)
end
js_tags_regexp = Regexp.union(js_tags)
- @js_base_folders.flat_map do |js_base_folder|
+ matching_js_files = @js_base_folders.flat_map do |js_base_folder|
Dir["#{js_base_folder}/**/*.{js,vue}"].select do |js_file|
file_content = File.read(js_file)
js_tags_regexp.match?(file_content)
end
end
+
+ write_array_to_file(predictive_tests_pathname, matching_js_files)
end
# Keep the files that are in the @view_base_folders folder
- def filter_files(changed_files)
- changed_files.select do |filename|
+ def filter_files
+ @_filter_files ||= changed_files.select do |filename|
filename.start_with?(*@view_base_folders) &&
File.exist?(filename)
end
@@ -69,6 +73,10 @@ module Tooling
def find_pattern_in_file(file, pattern)
File.read(file).scan(pattern).flatten.uniq
end
+
+ private
+
+ attr_reader :changed_files, :predictive_tests_pathname
end
end
end
diff --git a/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb b/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb
new file mode 100644
index 00000000000..6d840dcbd71
--- /dev/null
+++ b/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require_relative '../../../../lib/gitlab_edition'
+
+# Returns system specs files that are related to the Rails views files that were changed in the MR.
+module Tooling
+ module Mappings
+ class ViewToSystemSpecsMappings < Base
+ def initialize(changed_files_pathname, predictive_tests_pathname, view_base_folder: 'app/views')
+ @predictive_tests_pathname = predictive_tests_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @view_base_folders = folders_for_available_editions(view_base_folder)
+ end
+
+ def execute
+ found_system_specs = []
+
+ filter_files.each do |modified_view_file|
+ system_specs_exact_match = find_system_specs_exact_match(modified_view_file)
+ if system_specs_exact_match
+ found_system_specs << system_specs_exact_match
+ next
+ else
+ system_specs_parent_folder_match = find_system_specs_parent_folder_match(modified_view_file)
+ found_system_specs += system_specs_parent_folder_match unless system_specs_parent_folder_match.empty?
+ end
+ end
+
+ write_array_to_file(predictive_tests_pathname, found_system_specs.compact.uniq.sort)
+ end
+
+ private
+
+ attr_reader :changed_files, :predictive_tests_pathname, :view_base_folders
+
+ # Keep the views files that are in the @view_base_folders folder
+ def filter_files
+ @_filter_files ||= changed_files.select do |filename|
+ filename.start_with?(*view_base_folders) &&
+ File.basename(filename).end_with?('.html.haml') &&
+ File.exist?(filename)
+ end
+ end
+
+ def find_system_specs_exact_match(view_file)
+ potential_spec_file = to_feature_spec_folder(view_file).sub('.html.haml', '_spec.rb')
+
+ potential_spec_file if File.exist?(potential_spec_file)
+ end
+
+ def find_system_specs_parent_folder_match(view_file)
+ parent_system_specs_folder = File.dirname(to_feature_spec_folder(view_file))
+
+ Dir["#{parent_system_specs_folder}/**/*_spec.rb"]
+ end
+
+ # e.g. go from app/views/groups/merge_requests.html.haml to spec/features/groups/merge_requests.html.haml
+ def to_feature_spec_folder(view_file)
+ view_file.sub(%r{(ee/|jh/)?app/views}, '\1spec/features')
+ end
+ end
+ end
+end
diff --git a/tooling/lib/tooling/predictive_tests.rb b/tooling/lib/tooling/predictive_tests.rb
new file mode 100644
index 00000000000..503426b5520
--- /dev/null
+++ b/tooling/lib/tooling/predictive_tests.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require_relative 'find_changes'
+require_relative 'find_tests'
+require_relative 'mappings/graphql_base_type_mappings'
+require_relative 'mappings/js_to_system_specs_mappings'
+require_relative 'mappings/partial_to_views_mappings'
+require_relative 'mappings/view_to_js_mappings'
+require_relative 'mappings/view_to_system_specs_mappings'
+
+module Tooling
+ class PredictiveTests
+ REQUIRED_ENV_VARIABLES = %w[
+ RSPEC_CHANGED_FILES_PATH
+ RSPEC_MATCHING_TESTS_PATH
+ RSPEC_VIEWS_INCLUDING_PARTIALS_PATH
+ FRONTEND_FIXTURES_MAPPING_PATH
+ RSPEC_MATCHING_JS_FILES_PATH
+ ].freeze
+
+ def initialize
+ missing_env_variables = REQUIRED_ENV_VARIABLES.select { |key| ENV[key.to_s] == '' }
+ unless missing_env_variables.empty?
+ raise "[predictive tests] Missing ENV variable(s): #{missing_env_variables.join(',')}."
+ end
+
+ @rspec_changed_files_path = ENV['RSPEC_CHANGED_FILES_PATH']
+ @rspec_matching_tests_path = ENV['RSPEC_MATCHING_TESTS_PATH']
+ @rspec_views_including_partials_path = ENV['RSPEC_VIEWS_INCLUDING_PARTIALS_PATH']
+ @frontend_fixtures_mapping_path = ENV['FRONTEND_FIXTURES_MAPPING_PATH']
+ @rspec_matching_js_files_path = ENV['RSPEC_MATCHING_JS_FILES_PATH']
+ end
+
+ def execute
+ Tooling::FindChanges.new(
+ from: :api,
+ changed_files_pathname: rspec_changed_files_path
+ ).execute
+ Tooling::FindTests.new(rspec_changed_files_path, rspec_matching_tests_path).execute
+ Tooling::Mappings::PartialToViewsMappings.new(
+ rspec_changed_files_path, rspec_views_including_partials_path).execute
+ Tooling::FindTests.new(rspec_views_including_partials_path, rspec_matching_tests_path).execute
+ Tooling::Mappings::JsToSystemSpecsMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute
+ Tooling::Mappings::GraphqlBaseTypeMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute
+ Tooling::Mappings::ViewToSystemSpecsMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute
+ Tooling::FindChanges.new(
+ from: :changed_files,
+ changed_files_pathname: rspec_changed_files_path,
+ predictive_tests_pathname: rspec_matching_tests_path,
+ frontend_fixtures_mapping_pathname: frontend_fixtures_mapping_path
+ ).execute
+ Tooling::Mappings::ViewToJsMappings.new(rspec_changed_files_path, rspec_matching_js_files_path).execute
+ end
+
+ private
+
+ attr_reader :rspec_changed_files_path, :rspec_matching_tests_path, :rspec_views_including_partials_path,
+ :frontend_fixtures_mapping_path, :rspec_matching_js_files_path
+ end
+end
diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb
index eeda135f3ee..20e00763f65 100644
--- a/tooling/quality/test_level.rb
+++ b/tooling/quality/test_level.rb
@@ -18,6 +18,7 @@ module Quality
unit: %w[
bin
channels
+ components
config
contracts
db
@@ -54,7 +55,6 @@ module Quality
views
workers
tooling
- components
],
integration: %w[
commands
@@ -77,8 +77,8 @@ module Quality
@patterns[level] ||= "#{prefixes_for_pattern}spec/#{folders_pattern(level)}{,/**/}*#{suffix(level)}".freeze # rubocop:disable Style/RedundantFreeze
end
- def regexp(level)
- @regexps[level] ||= Regexp.new("#{prefixes_for_regex}spec/#{folders_regex(level)}").freeze
+ def regexp(level, start_with = false)
+ @regexps[level] ||= Regexp.new("#{'^' if start_with}#{prefixes_for_regex}spec/#{folders_regex(level)}").freeze
end
def level_for(file_path)