Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-02-20 16:49:51 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-02-20 16:49:51 +0300
commit71786ddc8e28fbd3cb3fcc4b3ff15e5962a1c82e (patch)
tree6a2d93ef3fb2d353bb7739e4b57e6541f51cdd71 /tooling
parenta7253423e3403b8c08f8a161e5937e1488f5f407 (diff)
Add latest changes from gitlab-org/gitlab@15-9-stable-eev15.9.0-rc42
Diffstat (limited to 'tooling')
-rwxr-xr-xtooling/bin/js_to_system_specs_mappings14
-rwxr-xr-xtooling/bin/view_to_js_mappings4
-rw-r--r--tooling/danger/config_files.rb23
-rw-r--r--tooling/danger/feature_flag.rb20
-rw-r--r--tooling/danger/product_intelligence.rb12
-rw-r--r--tooling/danger/project_helper.rb1
-rw-r--r--tooling/danger/specs.rb84
-rw-r--r--tooling/danger/stable_branch.rb69
-rw-r--r--tooling/danger/suggestor.rb63
-rw-r--r--tooling/lib/tooling/find_codeowners.rb2
-rw-r--r--tooling/lib/tooling/helm3_client.rb20
-rw-r--r--tooling/lib/tooling/kubernetes_client.rb11
-rw-r--r--tooling/lib/tooling/mappings/base.rb30
-rw-r--r--tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb61
-rw-r--r--tooling/lib/tooling/mappings/view_to_js_mappings.rb74
-rw-r--r--tooling/lib/tooling/parallel_rspec_runner.rb2
-rw-r--r--tooling/lib/tooling/test_map_packer.rb4
-rw-r--r--tooling/lib/tooling/view_to_js_mappings.rb77
-rw-r--r--tooling/quality/test_level.rb2
19 files changed, 380 insertions, 193 deletions
diff --git a/tooling/bin/js_to_system_specs_mappings b/tooling/bin/js_to_system_specs_mappings
new file mode 100755
index 00000000000..3e9d9cb4c5f
--- /dev/null
+++ b/tooling/bin/js_to_system_specs_mappings
@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/tooling/mappings/js_to_system_specs_mappings'
+
+changes = ARGV.shift
+matching_tests = ARGV.shift
+
+changed_files = File.read(changes).split(' ')
+matching_test_files = File.read(matching_tests).split(' ')
+
+system_tests = Tooling::Mappings::JsToSystemSpecsMappings.new.execute(changed_files)
+
+File.write(matching_tests, (matching_test_files + system_tests).join(' '))
diff --git a/tooling/bin/view_to_js_mappings b/tooling/bin/view_to_js_mappings
index 2cebb91892e..483003aac5e 100755
--- a/tooling/bin/view_to_js_mappings
+++ b/tooling/bin/view_to_js_mappings
@@ -1,10 +1,10 @@
#!/usr/bin/env ruby
# frozen_string_literal: true
-require_relative '../lib/tooling/view_to_js_mappings'
+require_relative '../lib/tooling/mappings/view_to_js_mappings'
changes = ARGV.shift
output_file = ARGV.shift
changed_files = File.read(changes).split(' ')
-File.write(output_file, Tooling::ViewToJsMappings.new.execute(changed_files).join(' '))
+File.write(output_file, Tooling::Mappings::ViewToJsMappings.new.execute(changed_files).join(' '))
diff --git a/tooling/danger/config_files.rb b/tooling/danger/config_files.rb
index 914605a3783..e165792471f 100644
--- a/tooling/danger/config_files.rb
+++ b/tooling/danger/config_files.rb
@@ -1,15 +1,14 @@
# frozen_string_literal: true
require 'yaml'
+require_relative 'suggestor'
module Tooling
module Danger
module ConfigFiles
- SUGGEST_INTRODUCED_BY_COMMENT = <<~SUGGEST_COMMENT
- ```suggestion
- introduced_by_url: %<url>
- ```
- SUGGEST_COMMENT
+ include ::Tooling::Danger::Suggestor
+
+ MISSING_INTRODUCED_BY_REGEX = /^\+?(?<attr_name>\s*introduced_by_url):\s*$/.freeze
CONFIG_DIRS = %w[
config/feature_flags
@@ -18,14 +17,12 @@ module Tooling
].freeze
def add_suggestion_for_missing_introduced_by_url
- new_config_files.each do |file_name|
- config_file_lines = project_helper.file_lines(file_name)
-
- config_file_lines.each_with_index do |added_line, i|
- next unless added_line =~ /^introduced_by_url:\s?$/
-
- markdown(format(SUGGEST_INTRODUCED_BY_COMMENT, url: helper.mr_web_url), file: file_name, line: i + 1)
- end
+ new_config_files.each do |filename|
+ add_suggestion(
+ filename: filename,
+ regex: MISSING_INTRODUCED_BY_REGEX,
+ replacement: "\\k<attr_name>: #{helper.mr_web_url}"
+ )
end
end
diff --git a/tooling/danger/feature_flag.rb b/tooling/danger/feature_flag.rb
index cef64e52af3..da0b7053af1 100644
--- a/tooling/danger/feature_flag.rb
+++ b/tooling/danger/feature_flag.rb
@@ -16,26 +16,22 @@ module Tooling
end
class Found
+ ATTRIBUTES = %w[name introduced_by_url rollout_issue_url milestone type group default_enabled].freeze
+
attr_reader :path
def initialize(path)
@path = path
end
- def raw
- @raw ||= File.read(path)
- end
-
- def group
- @group ||= yaml['group']
+ ATTRIBUTES.each do |attribute|
+ define_method(attribute) do
+ yaml[attribute]
+ end
end
- def default_enabled
- @default_enabled ||= yaml['default_enabled']
- end
-
- def rollout_issue_url
- @rollout_issue_url ||= yaml['rollout_issue_url']
+ def raw
+ @raw ||= File.read(path)
end
def group_match_mr_label?(mr_group_label)
diff --git a/tooling/danger/product_intelligence.rb b/tooling/danger/product_intelligence.rb
index 58e327408a1..d25f966504f 100644
--- a/tooling/danger/product_intelligence.rb
+++ b/tooling/danger/product_intelligence.rb
@@ -22,6 +22,11 @@ module Tooling
MSG
+ CHANGED_USAGE_DATA_MESSAGE = <<~MSG
+ Notice that implementing metrics directly in usage_data.rb has been deprecated. ([Deprecated Usage Metrics](https://docs.gitlab.com/ee/development/service_ping/usage_data.html#usage-data-metrics-guide))
+ Please use [Instrumentation Classes](https://docs.gitlab.com/ee/development/service_ping/metrics_instrumentation.html) instead.
+ MSG
+
WORKFLOW_LABELS = [
APPROVED_LABEL,
REVIEW_LABEL
@@ -47,6 +52,13 @@ module Tooling
helper.labels_to_add.concat(missing_labels) unless missing_labels.empty?
end
+ def check_usage_data_insertions!
+ usage_data_changes = helper.changed_lines("lib/gitlab/usage_data.rb")
+ return if usage_data_changes.none? { |change| change.start_with?("+") }
+
+ warn format(CHANGED_USAGE_DATA_MESSAGE)
+ end
+
private
def convert_to_table(items)
diff --git a/tooling/danger/project_helper.rb b/tooling/danger/project_helper.rb
index fbf102422aa..2a77ac337a2 100644
--- a/tooling/danger/project_helper.rb
+++ b/tooling/danger/project_helper.rb
@@ -12,6 +12,7 @@ module Tooling
sidekiq_queues
specialization_labels
specs
+ stable_branch_patch
z_metadata
].freeze
diff --git a/tooling/danger/specs.rb b/tooling/danger/specs.rb
index 6c0459a4344..f95a798d53e 100644
--- a/tooling/danger/specs.rb
+++ b/tooling/danger/specs.rb
@@ -1,12 +1,14 @@
# frozen_string_literal: true
+require_relative 'suggestor'
+
module Tooling
module Danger
module Specs
+ include ::Tooling::Danger::Suggestor
+
SPEC_FILES_REGEX = 'spec/'
EE_PREFIX = 'ee/'
- MATCH_WITH_ARRAY_REGEX = /(?<to>to\(?\s*)(?<matcher>match|eq)(?<expectation>[( ]?\[[^\]]+)/.freeze
- MATCH_WITH_ARRAY_REPLACEMENT = '\k<to>match_array\k<expectation>'
PROJECT_FACTORIES = %w[
:project
@@ -29,22 +31,18 @@ module Tooling
/x.freeze
PROJECT_FACTORY_REPLACEMENT = '\k<head>let_it_be\k<tail>'
- SUGGESTION_MARKDOWN = <<~SUGGESTION_MARKDOWN
- ```suggestion
- %<suggested_line>s
- ```
- SUGGESTION_MARKDOWN
-
- MATCH_WITH_ARRAY_SUGGESTION = <<~SUGGEST_COMMENT
- If order of the result is not important, please consider using `match_array` to avoid flakiness.
- SUGGEST_COMMENT
-
PROJECT_FACTORY_SUGGESTION = <<~SUGGEST_COMMENT
Project creations are very slow. Use `let_it_be`, `build` or `build_stubbed` if possible.
See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#optimize-factory-usage)
for background information and alternative options.
SUGGEST_COMMENT
+ MATCH_WITH_ARRAY_REGEX = /(?<to>to\(?\s*)(?<matcher>match|eq)(?<expectation>[( ]?\[(?=.*,)[^\]]+)/.freeze
+ MATCH_WITH_ARRAY_REPLACEMENT = '\k<to>match_array\k<expectation>'
+ MATCH_WITH_ARRAY_SUGGESTION = <<~SUGGEST_COMMENT
+ If order of the result is not important, please consider using `match_array` to avoid flakiness.
+ SUGGEST_COMMENT
+
RSPEC_TOP_LEVEL_DESCRIBE_REGEX = /^\+.?RSpec\.describe(.+)/.freeze
FEATURE_CATEGORY_SUGGESTION = <<~SUGGESTION_MARKDOWN
Consider adding `feature_category: <feature_category_name>` for this example if it is not set already.
@@ -69,19 +67,19 @@ module Tooling
def add_suggestions_for_match_with_array(filename)
add_suggestion(
- filename,
- MATCH_WITH_ARRAY_REGEX,
- MATCH_WITH_ARRAY_SUGGESTION,
- MATCH_WITH_ARRAY_REPLACEMENT
+ filename: filename,
+ regex: MATCH_WITH_ARRAY_REGEX,
+ replacement: MATCH_WITH_ARRAY_REPLACEMENT,
+ comment_text: MATCH_WITH_ARRAY_SUGGESTION
)
end
def add_suggestions_for_project_factory_usage(filename)
add_suggestion(
- filename,
- PROJECT_FACTORY_REGEX,
- PROJECT_FACTORY_SUGGESTION,
- PROJECT_FACTORY_REPLACEMENT
+ filename: filename,
+ regex: PROJECT_FACTORY_REGEX,
+ replacement: PROJECT_FACTORY_REPLACEMENT,
+ comment_text: PROJECT_FACTORY_SUGGESTION
)
end
@@ -103,53 +101,9 @@ module Tooling
suggested_line = file_lines[line_number]
- text = format(comment(FEATURE_CATEGORY_SUGGESTION), suggested_line: suggested_line)
- markdown(text, file: filename, line: line_number + 1)
+ markdown(comment(FEATURE_CATEGORY_SUGGESTION, suggested_line), file: filename, line: line_number.succ)
end
end
-
- private
-
- def added_lines_matching(filename, regex)
- helper.changed_lines(filename).grep(/\A\+( )?/).grep(regex)
- end
-
- def add_suggestion(filename, regex, comment_text, replacement = nil, exclude = nil)
- added_lines = added_lines_matching(filename, regex)
-
- return if added_lines.empty?
-
- spec_file_lines = project_helper.file_lines(filename)
-
- added_lines.each_with_object([]) do |added_line, processed_line_numbers|
- line_number = find_line_number(spec_file_lines, added_line.delete_prefix('+'), exclude_indexes: processed_line_numbers)
- next unless line_number
- next if !exclude.nil? && added_line.include?(exclude)
-
- processed_line_numbers << line_number
-
- suggested_line = spec_file_lines[line_number]
- suggested_line = suggested_line.gsub(regex, replacement) unless replacement.nil?
-
- text = format(comment(comment_text), suggested_line: suggested_line)
- markdown(text, file: filename, line: line_number.succ)
- end
- end
-
- def comment(comment_text)
- <<~COMMENT_BODY.chomp
- #{SUGGESTION_MARKDOWN}
- #{comment_text}
- COMMENT_BODY
- end
-
- def find_line_number(file_lines, searched_line, exclude_indexes: [])
- _, index = file_lines.each_with_index.find do |file_line, index|
- file_line == searched_line && !exclude_indexes.include?(index)
- end
-
- index
- end
end
end
end
diff --git a/tooling/danger/stable_branch.rb b/tooling/danger/stable_branch.rb
index 6c0b94b4f06..8fac1cc5fbf 100644
--- a/tooling/danger/stable_branch.rb
+++ b/tooling/danger/stable_branch.rb
@@ -6,6 +6,7 @@ module Tooling
VersionApiError = Class.new(StandardError)
STABLE_BRANCH_REGEX = %r{\A(?<version>\d+-\d+)-stable-ee\z}.freeze
+ FAILING_PACKAGE_AND_TEST_STATUSES = %w[manual canceled].freeze
# rubocop:disable Lint/MixedRegexpCaptureTypes
VERSION_REGEX = %r{
@@ -32,26 +33,69 @@ module Tooling
This branch is meant for backporting bug fixes. If this MR qualifies please add the `type::bug` label. #{MAINTENANCE_POLICY_MESSAGE}
MSG
- VERSION_ERROR_MESSAGE = <<~MSG
- Patches are only being accepted on the most recent 3 minor versions of GitLab. #{MAINTENANCE_POLICY_MESSAGE}
+ VERSION_WARNING_MESSAGE = <<~MSG
+ Backporting to older releases requires an [exception request process](https://docs.gitlab.com/ee/policy/maintenance.html#backporting-to-older-releases)
MSG
FAILED_VERSION_REQUEST_MESSAGE = <<~MSG
There was a problem checking if this is a qualified version for backporting. Re-running this job may fix the problem.
MSG
+ PIPELINE_EXPEDITE_ERROR_MESSAGE = <<~MSG
+ ~"pipeline:expedite" is not allowed on stable branches because it causes the `e2e:package-and-test` job to be skipped.
+ MSG
+
+ NEEDS_PACKAGE_AND_TEST_MESSAGE = <<~MSG
+ The `e2e:package-and-test` job is not present or needs to be triggered manually. Please start the `e2e:package-and-test`
+ job and re-run `danger-review`.
+ MSG
+
+ WARN_PACKAGE_AND_TEST_MESSAGE = <<~MSG
+ The `e2e:package-and-test` job needs to succeed or have approval from a Software Engineer in Test. See the section below
+ for more details.
+ MSG
+
# rubocop:disable Style/SignalException
def check!
- return unless stable_target_branch && !helper.security_mr?
+ return unless non_security_stable_branch?
fail FEATURE_ERROR_MESSAGE if has_feature_label?
fail BUG_ERROR_MESSAGE unless has_bug_label?
- fail VERSION_ERROR_MESSAGE unless targeting_patchable_version?
+
+ warn VERSION_WARNING_MESSAGE unless targeting_patchable_version?
+
+ return if has_flaky_failure_label? || has_only_documentation_changes?
+
+ fail PIPELINE_EXPEDITE_ERROR_MESSAGE if has_pipeline_expedite_label?
+
+ status = package_and_test_status
+
+ if status.nil? || FAILING_PACKAGE_AND_TEST_STATUSES.include?(status) # rubocop:disable Style/GuardClause
+ fail NEEDS_PACKAGE_AND_TEST_MESSAGE
+ else
+ warn WARN_PACKAGE_AND_TEST_MESSAGE unless status == 'success'
+ end
end
# rubocop:enable Style/SignalException
+ def non_security_stable_branch?
+ !!stable_target_branch && !helper.security_mr?
+ end
+
private
+ def package_and_test_status
+ mr_head_pipeline_id = gitlab.mr_json.dig('head_pipeline', 'id')
+ return unless mr_head_pipeline_id
+
+ pipeline_bridges = gitlab.api.pipeline_bridges(helper.mr_target_project_id, mr_head_pipeline_id)
+ package_and_test_pipeline = pipeline_bridges&.find { |j| j['name'] == 'e2e:package-and-test' }
+
+ return unless package_and_test_pipeline
+
+ package_and_test_pipeline['status']
+ end
+
def stable_target_branch
helper.mr_target_branch.match(STABLE_BRANCH_REGEX)
end
@@ -64,12 +108,27 @@ module Tooling
helper.mr_has_labels?('type::bug')
end
+ def has_pipeline_expedite_label?
+ helper.mr_has_labels?('pipeline:expedite')
+ end
+
+ def has_flaky_failure_label?
+ helper.mr_has_labels?('failure::flaky-test')
+ end
+
+ def has_only_documentation_changes?
+ categories_changed = helper.changes_by_category.keys
+ return false unless categories_changed.size == 1
+ return true if categories_changed.first == :docs
+
+ false
+ end
+
def targeting_patchable_version?
raise VersionApiError if last_three_minor_versions.empty?
last_three_minor_versions.include?(targeted_version)
rescue VersionApiError
- # don't fail the job since we do not know the recent versions
warn FAILED_VERSION_REQUEST_MESSAGE
true
end
diff --git a/tooling/danger/suggestor.rb b/tooling/danger/suggestor.rb
new file mode 100644
index 00000000000..ffda98e67d0
--- /dev/null
+++ b/tooling/danger/suggestor.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+module Tooling
+ module Danger
+ module Suggestor
+ # For file lines matching `regex` adds suggestion `replacement` with `comment_text` added.
+ def add_suggestion(filename:, regex:, replacement: nil, comment_text: nil, exclude: nil)
+ added_lines = added_lines_matching(filename, regex)
+
+ return if added_lines.empty?
+
+ file_lines = project_helper.file_lines(filename)
+
+ added_lines.each_with_object([]) do |added_line, processed_line_numbers|
+ line_number = find_line_number(file_lines, added_line.delete_prefix('+'),
+exclude_indexes: processed_line_numbers)
+
+ next unless line_number
+ next if !exclude.nil? && added_line.include?(exclude)
+
+ processed_line_numbers << line_number
+
+ if replacement
+ suggestion_text = file_lines[line_number]
+ suggestion_text = suggestion_text.gsub(regex, replacement)
+ end
+
+ markdown(comment(comment_text, suggestion_text), file: filename, line: line_number.succ)
+ end
+ end
+
+ private
+
+ def added_lines_matching(filename, regex)
+ helper.changed_lines(filename).grep(/\A\+( )?/).grep(regex)
+ end
+
+ def find_line_number(file_lines, searched_line, exclude_indexes: [])
+ _, index = file_lines.each_with_index.find do |file_line, index|
+ file_line == searched_line && !exclude_indexes.include?(index) # rubocop:disable Rails/NegateInclude
+ end
+
+ index
+ end
+
+ def comment(comment_text = nil, suggested_line = nil)
+ if suggested_line
+ suggestion_text = <<~SUGGESTION
+ ```suggestion
+ %<suggested_line>s
+ ```
+ SUGGESTION
+ end
+
+ comment_body = <<~COMMENT_BODY.chomp
+ #{suggestion_text}
+ #{comment_text}
+ COMMENT_BODY
+
+ format(comment_body.chomp, suggested_line: suggested_line)
+ end
+ end
+ end
+end
diff --git a/tooling/lib/tooling/find_codeowners.rb b/tooling/lib/tooling/find_codeowners.rb
index 6a90f86eecc..cc37d4db1ec 100644
--- a/tooling/lib/tooling/find_codeowners.rb
+++ b/tooling/lib/tooling/find_codeowners.rb
@@ -89,7 +89,7 @@ module Tooling
end
def consolidate_paths(matched_files)
- matched_files.group_by(&File.method(:dirname)).flat_map do |dir, files|
+ matched_files.group_by { |file| File.dirname(file) }.flat_map do |dir, files|
# First line is the dir itself
if find_dir_maxdepth_1(dir).lines.drop(1).sort == files.sort
"#{dir}\n"
diff --git a/tooling/lib/tooling/helm3_client.rb b/tooling/lib/tooling/helm3_client.rb
index d4e7faa802e..9059387351a 100644
--- a/tooling/lib/tooling/helm3_client.rb
+++ b/tooling/lib/tooling/helm3_client.rb
@@ -8,34 +8,32 @@ module Tooling
class Helm3Client
CommandFailedError = Class.new(StandardError)
- attr_reader :namespace
-
RELEASE_JSON_ATTRIBUTES = %w[name revision updated status chart app_version namespace].freeze
PAGINATION_SIZE = 256 # Default helm list pagination size
- Release = Struct.new(:name, :revision, :last_update, :status, :chart, :app_version, :namespace) do
+ Release = Struct.new(:name, :namespace, :revision, :updated, :status, :chart, :app_version, keyword_init: true) do
def revision
@revision ||= self[:revision].to_i
end
def last_update
- @last_update ||= self[:last_update] ? Time.parse(self[:last_update]) : nil
+ @last_update ||= self[:updated] ? Time.parse(self[:updated]) : nil
end
end
# A single page of data and the corresponding page number.
Page = Struct.new(:releases, :number)
- def initialize(namespace:)
- @namespace = namespace
- end
-
def releases(args: [])
each_release(args)
end
- def delete(release_name:)
- run_command(['uninstall', release_name])
+ def delete(release_name:, namespace: nil)
+ release_name = Array(release_name)
+
+ release_name.each do |release|
+ run_command(['uninstall', '--namespace', (namespace || release), release])
+ end
end
private
@@ -66,7 +64,7 @@ module Tooling
releases = JSON.parse(response) # rubocop:disable Gitlab/Json
releases.map do |release|
- Release.new(*release.values_at(*RELEASE_JSON_ATTRIBUTES))
+ Release.new(release.slice(*RELEASE_JSON_ATTRIBUTES))
end
rescue ::JSON::ParserError => ex
puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output
diff --git a/tooling/lib/tooling/kubernetes_client.rb b/tooling/lib/tooling/kubernetes_client.rb
index 1d7b924e2c3..ab914db5777 100644
--- a/tooling/lib/tooling/kubernetes_client.rb
+++ b/tooling/lib/tooling/kubernetes_client.rb
@@ -6,8 +6,9 @@ require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen)
module Tooling
class KubernetesClient
- RESOURCE_LIST = 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd'
- CommandFailedError = Class.new(StandardError)
+ RESOURCE_LIST = 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd'
+ K8S_ALLOWED_NAMESPACES_REGEX = /^review-(?!apps).+/.freeze
+ CommandFailedError = Class.new(StandardError)
attr_reader :namespace
@@ -129,14 +130,16 @@ module Tooling
command = [
'get',
'namespace',
- "-l tls=review-apps-tls", # Get only namespaces used for review-apps
"--sort-by='{.metadata.creationTimestamp}'",
'-o json'
]
response = run_command(command)
- resources_created_before_date(response, created_before)
+ stale_namespaces = resources_created_before_date(response, created_before)
+
+ # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby.
+ stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) }
end
def resources_created_before_date(response, date)
diff --git a/tooling/lib/tooling/mappings/base.rb b/tooling/lib/tooling/mappings/base.rb
new file mode 100644
index 00000000000..93d3a967114
--- /dev/null
+++ b/tooling/lib/tooling/mappings/base.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative '../../../../lib/gitlab_edition'
+
+# Returns system specs files that are related to the JS files that were changed in the MR.
+module Tooling
+ module Mappings
+ class Base
+ # Input: A list of space-separated files
+ # Output: A list of space-separated specs files (JS, Ruby, ...)
+ def execute(changed_files)
+ raise "Not Implemented"
+ end
+
+ # Input: A list of space-separated files
+ # Output: array/hash of files
+ def filter_files(changed_files)
+ raise "Not Implemented"
+ end
+
+ # Input: A folder
+ # Output: An array of folders, each prefixed with a GitLab edition
+ def folders_for_available_editions(base_folder)
+ foss_prefix = base_folder
+ extension_prefixes = ::GitlabEdition.extensions.map { |prefix| "#{prefix}/#{foss_prefix}" }
+ [foss_prefix, *extension_prefixes]
+ end
+ end
+ end
+end
diff --git a/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb
new file mode 100644
index 00000000000..365e466011b
--- /dev/null
+++ b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'active_support/inflector'
+
+require_relative 'base'
+require_relative '../../../../lib/gitlab_edition'
+
+# Returns system specs files that are related to the JS files that were changed in the MR.
+module Tooling
+ module Mappings
+ class JsToSystemSpecsMappings < Base
+ def initialize(js_base_folder: 'app/assets/javascripts', system_specs_base_folder: 'spec/features')
+ @js_base_folder = js_base_folder
+ @js_base_folders = folders_for_available_editions(js_base_folder)
+ @system_specs_base_folder = system_specs_base_folder
+
+ # Cannot be extracted to a constant, as it depends on a variable
+ @first_js_folder_extract_regexp = %r{
+ (?:.*/)? # Skips the GitLab edition (e.g. ee/, jh/)
+ #{@js_base_folder}/ # Most likely app/assets/javascripts/
+ ([\w-]*) # Captures the first folder
+ }x
+ end
+
+ def execute(changed_files)
+ filter_files(changed_files).flat_map do |edition, js_files|
+ js_keywords_regexp = Regexp.union(construct_js_keywords(js_files))
+
+ system_specs_for_edition(edition).select do |system_spec_file|
+ system_spec_file if js_keywords_regexp.match?(system_spec_file)
+ end
+ end
+ end
+
+ # Keep the files that are in the @js_base_folders folders
+ #
+ # Returns a hash, where the key is the GitLab edition, and the values the JS specs
+ def filter_files(changed_files)
+ selected_files = changed_files.select do |filename|
+ filename.start_with?(*@js_base_folders) && File.exist?(filename)
+ end
+
+ selected_files.group_by { |filename| filename[/^#{Regexp.union(::GitlabEdition.extensions)}/] }
+ end
+
+ # Extract keywords in the JS filenames to be used for searching matching system specs
+ def construct_js_keywords(js_files)
+ js_files.map do |js_file|
+ filename = js_file.scan(@first_js_folder_extract_regexp).flatten.first
+ filename.singularize
+ end.uniq
+ end
+
+ def system_specs_for_edition(edition)
+ all_files_in_folders_glob = File.join(@system_specs_base_folder, '**', '*')
+ all_files_in_folders_glob = File.join(edition, all_files_in_folders_glob) if edition
+ Dir[all_files_in_folders_glob].select { |f| File.file?(f) }
+ end
+ end
+ end
+end
diff --git a/tooling/lib/tooling/mappings/view_to_js_mappings.rb b/tooling/lib/tooling/mappings/view_to_js_mappings.rb
new file mode 100644
index 00000000000..db80eb9bfe8
--- /dev/null
+++ b/tooling/lib/tooling/mappings/view_to_js_mappings.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+require_relative '../../../../lib/gitlab_edition'
+
+# Returns JS files that are related to the Rails views files that were changed in the MR.
+module Tooling
+ module Mappings
+ class ViewToJsMappings < Base
+ # The HTML attribute value pattern we're looking for to match an HTML file to a JS file.
+ HTML_ATTRIBUTE_VALUE_REGEXP = /js-[-\w]+/.freeze
+
+ # Search for Rails partials included in an HTML file
+ RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exist)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze
+
+ def initialize(view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts')
+ @view_base_folders = folders_for_available_editions(view_base_folder)
+ @js_base_folders = folders_for_available_editions(js_base_folder)
+ end
+
+ def execute(changed_files)
+ changed_view_files = filter_files(changed_files)
+
+ partials = changed_view_files.flat_map do |file|
+ find_partials(file)
+ end
+
+ files_to_scan = changed_view_files + partials
+ js_tags = files_to_scan.flat_map do |file|
+ find_pattern_in_file(file, HTML_ATTRIBUTE_VALUE_REGEXP)
+ end
+ js_tags_regexp = Regexp.union(js_tags)
+
+ @js_base_folders.flat_map do |js_base_folder|
+ Dir["#{js_base_folder}/**/*.{js,vue}"].select do |js_file|
+ file_content = File.read(js_file)
+ js_tags_regexp.match?(file_content)
+ end
+ end
+ end
+
+ # Keep the files that are in the @view_base_folders folder
+ def filter_files(changed_files)
+ changed_files.select do |filename|
+ filename.start_with?(*@view_base_folders) &&
+ File.exist?(filename)
+ end
+ end
+
+ # Note: We only search for partials with depth 1. We don't do recursive search, as
+ # it is probably not necessary for a first iteration.
+ def find_partials(file)
+ partial_paths = find_pattern_in_file(file, RAILS_PARTIAL_INVOCATION_REGEXP)
+ partial_paths.flat_map do |partial_path|
+ view_file_folder = File.dirname(file)
+ partial_relative_folder = File.dirname(partial_path)
+
+ dirname =
+ if partial_relative_folder == '.' # The partial is in the same folder as the HTML file
+ view_file_folder
+ else
+ File.join(view_file_folder, partial_relative_folder)
+ end
+
+ Dir["#{dirname}/_#{File.basename(partial_path)}.*"]
+ end
+ end
+
+ def find_pattern_in_file(file, pattern)
+ File.read(file).scan(pattern).flatten.uniq
+ end
+ end
+ end
+end
diff --git a/tooling/lib/tooling/parallel_rspec_runner.rb b/tooling/lib/tooling/parallel_rspec_runner.rb
index b482160d3c0..b1ddc91e831 100644
--- a/tooling/lib/tooling/parallel_rspec_runner.rb
+++ b/tooling/lib/tooling/parallel_rspec_runner.rb
@@ -43,6 +43,8 @@ module Tooling
return
end
+ Knapsack.logger.info "Running command: #{rspec_command.join(' ')}"
+
exec(*rspec_command)
end
diff --git a/tooling/lib/tooling/test_map_packer.rb b/tooling/lib/tooling/test_map_packer.rb
index 151ce88111f..15191e35c54 100644
--- a/tooling/lib/tooling/test_map_packer.rb
+++ b/tooling/lib/tooling/test_map_packer.rb
@@ -6,11 +6,11 @@ module Tooling
MARKER = 1
def pack(map)
- map.transform_values(&method(:create_tree_from_tests))
+ map.transform_values { |tests| create_tree_from_tests(tests) }
end
def unpack(compact_map)
- compact_map.transform_values(&method(:retrieve_tests_from_tree))
+ compact_map.transform_values { |tree| retrieve_tests_from_tree(tree) }
end
private
diff --git a/tooling/lib/tooling/view_to_js_mappings.rb b/tooling/lib/tooling/view_to_js_mappings.rb
deleted file mode 100644
index 76704a04469..00000000000
--- a/tooling/lib/tooling/view_to_js_mappings.rb
+++ /dev/null
@@ -1,77 +0,0 @@
-# frozen_string_literal: true
-
-require_relative '../../../lib/gitlab_edition'
-
-# Returns JS files that are related to the Rails views files that were changed in the MR.
-module Tooling
- class ViewToJsMappings
- # The HTML attribute value pattern we're looking for to match an HTML file to a JS file.
- HTML_ATTRIBUTE_VALUE_REGEXP = /js-[-\w]+/.freeze
-
- # Search for Rails partials included in an HTML file
- RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exist)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze
-
- def initialize(view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts')
- @view_base_folders = folders_for_available_editions(view_base_folder)
- @js_base_folders = folders_for_available_editions(js_base_folder)
- end
-
- def execute(changed_files)
- changed_view_files = view_files(changed_files)
-
- partials = changed_view_files.flat_map do |file|
- find_partials(file)
- end
-
- files_to_scan = changed_view_files + partials
- js_tags = files_to_scan.flat_map do |file|
- find_pattern_in_file(file, HTML_ATTRIBUTE_VALUE_REGEXP)
- end
- js_tags_regexp = Regexp.union(js_tags)
-
- @js_base_folders.flat_map do |js_base_folder|
- Dir["#{js_base_folder}/**/*.{js,vue}"].select do |js_file|
- file_content = File.read(js_file)
- js_tags_regexp.match?(file_content)
- end
- end
- end
-
- # Keep the files that are in the @view_base_folders folder
- def view_files(changed_files)
- changed_files.select do |filename|
- filename.start_with?(*@view_base_folders) &&
- File.exist?(filename)
- end
- end
-
- def folders_for_available_editions(base_folder)
- foss_prefix = base_folder
- extension_prefixes = ::GitlabEdition.extensions.map { |prefix| "#{prefix}/#{foss_prefix}" }
- [foss_prefix, *extension_prefixes]
- end
-
- # Note: We only search for partials with depth 1. We don't do recursive search, as
- # it is probably not necessary for a first iteration.
- def find_partials(file)
- partial_paths = find_pattern_in_file(file, RAILS_PARTIAL_INVOCATION_REGEXP)
- partial_paths.flat_map do |partial_path|
- view_file_folder = File.dirname(file)
- partial_relative_folder = File.dirname(partial_path)
-
- dirname =
- if partial_relative_folder == '.' # The partial is in the same folder as the HTML file
- view_file_folder
- else
- File.join(view_file_folder, partial_relative_folder)
- end
-
- Dir["#{dirname}/_#{File.basename(partial_path)}.*"]
- end
- end
-
- def find_pattern_in_file(file, pattern)
- File.read(file).scan(pattern).flatten.uniq
- end
- end
-end
diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb
index 31196290de4..eeda135f3ee 100644
--- a/tooling/quality/test_level.rb
+++ b/tooling/quality/test_level.rb
@@ -115,7 +115,7 @@ module Quality
def prefixes_for_regex
return '' if prefixes.empty?
- regex_prefix = prefixes.map(&Regexp.method(:escape)).join('|')
+ regex_prefix = prefixes.map { |prefix| Regexp.escape(prefix) }.join('|')
"(#{regex_prefix})"
end