Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-05-17 19:05:49 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-05-17 19:05:49 +0300
commit43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch)
treedceebdc68925362117480a5d672bcff122fb625b /tooling
parent20c84b99005abd1c82101dfeff264ac50d2df211 (diff)
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc42
Diffstat (limited to 'tooling')
-rwxr-xr-xtooling/bin/find_changes82
-rwxr-xr-xtooling/bin/find_only_allowed_files_changes12
-rwxr-xr-xtooling/bin/find_tests19
-rwxr-xr-xtooling/bin/gettext_extractor29
-rwxr-xr-xtooling/bin/js_to_system_specs_mappings14
-rwxr-xr-xtooling/bin/predictive_tests6
-rwxr-xr-xtooling/bin/view_to_js_mappings10
-rw-r--r--tooling/danger/analytics_instrumentation.rb (renamed from tooling/danger/product_intelligence.rb)22
-rw-r--r--tooling/danger/database_dictionary.rb57
-rw-r--r--tooling/danger/feature_flag.rb6
-rw-r--r--tooling/danger/multiversion.rb35
-rw-r--r--tooling/danger/project_helper.rb38
-rw-r--r--tooling/danger/sidekiq_args.rb59
-rw-r--r--tooling/danger/sidekiq_queues.rb2
-rw-r--r--tooling/danger/specs.rb86
-rw-r--r--tooling/danger/specs/feature_category_suggestion.rb42
-rw-r--r--tooling/danger/specs/match_with_array_suggestion.rb17
-rw-r--r--tooling/danger/specs/project_factory_suggestion.rb46
-rw-r--r--tooling/danger/stable_branch.rb18
-rw-r--r--tooling/danger/suggestion.rb39
-rw-r--r--tooling/docs/deprecation_handling.rb7
-rw-r--r--tooling/graphql/docs/helper.rb2
-rw-r--r--tooling/lib/tooling/fast_quarantine.rb57
-rwxr-xr-xtooling/lib/tooling/find_changes.rb109
-rw-r--r--tooling/lib/tooling/find_codeowners.rb6
-rw-r--r--tooling/lib/tooling/find_files_using_feature_flags.rb48
-rw-r--r--tooling/lib/tooling/find_tests.rb31
-rw-r--r--tooling/lib/tooling/gettext_extractor.rb111
-rw-r--r--tooling/lib/tooling/helpers/file_handler.rb31
-rw-r--r--tooling/lib/tooling/helpers/predictive_tests_helper.rb (renamed from tooling/lib/tooling/mappings/base.rb)17
-rw-r--r--tooling/lib/tooling/kubernetes_client.rb141
-rw-r--r--tooling/lib/tooling/mappings/graphql_base_type_mappings.rb121
-rw-r--r--tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb33
-rw-r--r--tooling/lib/tooling/mappings/partial_to_views_mappings.rb107
-rw-r--r--tooling/lib/tooling/mappings/view_to_js_mappings.rb38
-rw-r--r--tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb66
-rw-r--r--tooling/lib/tooling/parallel_rspec_runner.rb99
-rw-r--r--tooling/lib/tooling/predictive_tests.rb62
-rw-r--r--tooling/lib/tooling/test_map_generator.rb4
-rw-r--r--tooling/quality/test_level.rb6
-rw-r--r--tooling/rspec_flaky/config.rb4
41 files changed, 1287 insertions, 452 deletions
diff --git a/tooling/bin/find_changes b/tooling/bin/find_changes
deleted file mode 100755
index 38e1f363dd9..00000000000
--- a/tooling/bin/find_changes
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'gitlab'
-
-class FindChanges # rubocop:disable Gitlab/NamespacedClass
- def initialize(output_file:, matched_tests_file: nil, frontend_fixtures_mapping_path: nil)
- @gitlab_token = ENV.fetch('PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE', '')
- @gitlab_endpoint = ENV.fetch('CI_API_V4_URL')
- @mr_project_path = ENV.fetch('CI_MERGE_REQUEST_PROJECT_PATH')
- @mr_iid = ENV.fetch('CI_MERGE_REQUEST_IID')
- @output_file = output_file
- @matched_tests_file = matched_tests_file
- @frontend_fixtures_mapping_path = frontend_fixtures_mapping_path
- end
-
- def execute
- add_frontend_fixture_files!
-
- File.write(output_file, file_changes.join(' '))
- end
-
- private
-
- def add_frontend_fixture_files?
- matched_tests_file && frontend_fixtures_mapping_path
- end
-
- def add_frontend_fixture_files!
- return unless add_frontend_fixture_files?
-
- # If we have a `test file -> JSON frontend fixture` mapping file, we add the files JSON frontend fixtures
- # files to the list of changed files so that Jest can automatically run the dependent tests thanks to --findRelatedTests
- test_files.each do |test_file|
- file_changes.concat(frontend_fixtures_mapping[test_file]) if frontend_fixtures_mapping.key?(test_file)
- end
- end
-
- def file_changes
- @file_changes ||=
- if File.exist?(output_file)
- File.read(output_file).split(' ')
- else
- Gitlab.configure do |config|
- config.endpoint = gitlab_endpoint
- config.private_token = gitlab_token
- end
-
- mr_changes.changes.flat_map do |change|
- change.to_h.values_at('old_path', 'new_path')
- end.uniq
- end
- end
-
- def mr_changes
- @mr_changes ||= Gitlab.merge_request_changes(mr_project_path, mr_iid)
- end
-
- def test_files
- return [] if !matched_tests_file || !File.exist?(matched_tests_file)
-
- File.read(matched_tests_file).split(' ')
- end
-
- def frontend_fixtures_mapping
- return {} if !frontend_fixtures_mapping_path || !File.exist?(frontend_fixtures_mapping_path)
-
- JSON.parse(File.read(frontend_fixtures_mapping_path)) # rubocop:disable Gitlab/Json
- end
-
- attr_reader :gitlab_token, :gitlab_endpoint, :mr_project_path, :mr_iid, :output_file, :matched_tests_file, :frontend_fixtures_mapping_path
-end
-
-output_file = ARGV.shift
-raise ArgumentError, "An path to an output file must be given as first argument of #{__FILE__}." if output_file.nil?
-
-matched_tests_file = ARGV.shift
-frontend_fixtures_mapping_path = ARGV.shift
-
-FindChanges
- .new(output_file: output_file, matched_tests_file: matched_tests_file, frontend_fixtures_mapping_path: frontend_fixtures_mapping_path)
- .execute
diff --git a/tooling/bin/find_only_allowed_files_changes b/tooling/bin/find_only_allowed_files_changes
new file mode 100755
index 00000000000..c40048c66fa
--- /dev/null
+++ b/tooling/bin/find_only_allowed_files_changes
@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/tooling/find_changes'
+
+if Tooling::FindChanges.new(from: :api).only_allowed_files_changed
+ puts "Only files with extensions #{ALLOWED_FILE_TYPES.join(', ')} were changed"
+ exit 0
+else
+ puts "Changes were made to files with extensions other than #{ALLOWED_FILE_TYPES.join(', ')}"
+ exit 1
+end
diff --git a/tooling/bin/find_tests b/tooling/bin/find_tests
deleted file mode 100755
index 33834064f36..00000000000
--- a/tooling/bin/find_tests
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require 'test_file_finder'
-
-changes = ARGV.shift
-output_file = ARGV.shift
-
-changed_files = File.read(changes).split(' ')
-
-tff = TestFileFinder::FileFinder.new(paths: changed_files).tap do |file_finder|
- file_finder.use TestFileFinder::MappingStrategies::PatternMatching.load('tests.yml')
-
- if ENV['RSPEC_TESTS_MAPPING_ENABLED']
- file_finder.use TestFileFinder::MappingStrategies::DirectMatching.load_json(ENV['RSPEC_TESTS_MAPPING_PATH'])
- end
-end
-
-File.write(output_file, tff.test_files.uniq.join(' '))
diff --git a/tooling/bin/gettext_extractor b/tooling/bin/gettext_extractor
new file mode 100755
index 00000000000..39f029616df
--- /dev/null
+++ b/tooling/bin/gettext_extractor
@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/tooling/gettext_extractor'
+
+pot_file = ARGV.shift
+
+if !pot_file || !Dir.exist?(File.dirname(pot_file))
+ abort <<~MSG
+ Please provide a target file name as the first argument, e.g.
+ #{$PROGRAM_NAME} locale/gitlab.pot
+ MSG
+end
+
+puts <<~MSG
+ Extracting translatable strings from source files...
+MSG
+
+root_dir = File.expand_path('../../', __dir__)
+
+extractor = Tooling::GettextExtractor.new(
+ glob_base: root_dir
+)
+
+File.write(pot_file, extractor.generate_pot)
+
+puts <<~MSG
+ All done. Please commit the changes to `#{pot_file}`.
+MSG
diff --git a/tooling/bin/js_to_system_specs_mappings b/tooling/bin/js_to_system_specs_mappings
deleted file mode 100755
index 3e9d9cb4c5f..00000000000
--- a/tooling/bin/js_to_system_specs_mappings
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require_relative '../lib/tooling/mappings/js_to_system_specs_mappings'
-
-changes = ARGV.shift
-matching_tests = ARGV.shift
-
-changed_files = File.read(changes).split(' ')
-matching_test_files = File.read(matching_tests).split(' ')
-
-system_tests = Tooling::Mappings::JsToSystemSpecsMappings.new.execute(changed_files)
-
-File.write(matching_tests, (matching_test_files + system_tests).join(' '))
diff --git a/tooling/bin/predictive_tests b/tooling/bin/predictive_tests
new file mode 100755
index 00000000000..61543494a69
--- /dev/null
+++ b/tooling/bin/predictive_tests
@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative '../lib/tooling/predictive_tests'
+
+Tooling::PredictiveTests.new.execute
diff --git a/tooling/bin/view_to_js_mappings b/tooling/bin/view_to_js_mappings
deleted file mode 100755
index 483003aac5e..00000000000
--- a/tooling/bin/view_to_js_mappings
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require_relative '../lib/tooling/mappings/view_to_js_mappings'
-
-changes = ARGV.shift
-output_file = ARGV.shift
-changed_files = File.read(changes).split(' ')
-
-File.write(output_file, Tooling::Mappings::ViewToJsMappings.new.execute(changed_files).join(' '))
diff --git a/tooling/danger/product_intelligence.rb b/tooling/danger/analytics_instrumentation.rb
index d25f966504f..ce5ee55e3ee 100644
--- a/tooling/danger/product_intelligence.rb
+++ b/tooling/danger/analytics_instrumentation.rb
@@ -3,13 +3,13 @@
module Tooling
module Danger
- module ProductIntelligence
+ module AnalyticsInstrumentation
METRIC_DIRS = %w[lib/gitlab/usage/metrics/instrumentations ee/lib/gitlab/usage/metrics/instrumentations].freeze
- APPROVED_LABEL = 'product intelligence::approved'
- REVIEW_LABEL = 'product intelligence::review pending'
+ APPROVED_LABEL = 'analytics instrumentation::approved'
+ REVIEW_LABEL = 'analytics instrumentation::review pending'
CHANGED_FILES_MESSAGE = <<~MSG
- For the following files, a review from the [Data team and Product Intelligence team](https://gitlab.com/groups/gitlab-org/analytics-section/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) is recommended
- Please check the ~"product intelligence" [Service Ping guide](https://docs.gitlab.com/ee/development/service_ping/) or the [Snowplow guide](https://docs.gitlab.com/ee/development/snowplow/).
+ For the following files, a review from the [Data team and Analytics Instrumentation team](https://gitlab.com/groups/gitlab-org/analytics-section/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) is recommended
+ Please check the ~"analytics instrumentation" [Service Ping guide](https://docs.gitlab.com/ee/development/service_ping/) or the [Snowplow guide](https://docs.gitlab.com/ee/development/snowplow/).
For MR review guidelines, see the [Service Ping review guidelines](https://docs.gitlab.com/ee/development/service_ping/review_guidelines.html) or the [Snowplow review guidelines](https://docs.gitlab.com/ee/development/snowplow/review_guidelines.html).
@@ -18,7 +18,7 @@ module Tooling
MSG
CHANGED_SCOPE_MESSAGE = <<~MSG
- The following metrics could be affected by the modified scopes and require ~"product intelligence" review:
+ The following metrics could be affected by the modified scopes and require ~"analytics instrumentation" review:
MSG
@@ -33,13 +33,13 @@ module Tooling
].freeze
def check!
- # exit if not matching files or if no product intelligence labels
- product_intelligence_paths_to_review = helper.changes_by_category[:product_intelligence]
+ analytics_instrumentation_paths_to_review = helper.changes.by_category(:analytics_instrumentation).files
+
labels_to_add = missing_labels
- return if product_intelligence_paths_to_review.empty? || skip_review?
+ return if analytics_instrumentation_paths_to_review.empty? || skip_review?
- warn format(CHANGED_FILES_MESSAGE, changed_files: helper.markdown_list(product_intelligence_paths_to_review)) unless has_approved_label?
+ warn format(CHANGED_FILES_MESSAGE, changed_files: helper.markdown_list(analytics_instrumentation_paths_to_review)) unless has_approved_label?
helper.labels_to_add.concat(labels_to_add) unless labels_to_add.empty?
end
@@ -110,7 +110,7 @@ module Tooling
return [] unless helper.ci?
labels = []
- labels << 'product intelligence' unless helper.mr_has_labels?('product intelligence')
+ labels << 'analytics instrumentation' unless helper.mr_has_labels?('analytics instrumentation')
labels << REVIEW_LABEL unless has_workflow_labels?
labels
diff --git a/tooling/danger/database_dictionary.rb b/tooling/danger/database_dictionary.rb
new file mode 100644
index 00000000000..8776532ff84
--- /dev/null
+++ b/tooling/danger/database_dictionary.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+module Tooling
+ module Danger
+ module DatabaseDictionary
+ DICTIONARY_PATH_REGEXP = %r{db/docs/.*\.yml}
+
+ # `change_type` can be:
+ # - :added
+ # - :modified
+ # - :deleted
+ def database_dictionary_files(change_type:)
+ files = helper.public_send("#{change_type}_files") # rubocop:disable GitlabSecurity/PublicSend
+
+ files.filter_map { |path| Found.new(path) if path =~ DICTIONARY_PATH_REGEXP }
+ end
+
+ class Found
+ ATTRIBUTES = %w[
+ table_name classes feature_categories description introduced_by_url milestone gitlab_schema
+ ].freeze
+
+ attr_reader :path
+
+ def initialize(path)
+ @path = path
+ end
+
+ ATTRIBUTES.each do |attribute|
+ define_method(attribute) do
+ yaml[attribute]
+ end
+ end
+
+ def raw
+ @raw ||= File.read(path)
+ end
+
+ def ci_schema?
+ gitlab_schema == 'gitlab_ci'
+ end
+
+ def main_schema?
+ gitlab_schema == 'gitlab_main'
+ end
+
+ private
+
+ def yaml
+ @yaml ||= YAML.safe_load(raw)
+ end
+ end
+ end
+ end
+end
diff --git a/tooling/danger/feature_flag.rb b/tooling/danger/feature_flag.rb
index da0b7053af1..3fb20c561af 100644
--- a/tooling/danger/feature_flag.rb
+++ b/tooling/danger/feature_flag.rb
@@ -15,6 +15,12 @@ module Tooling
files.select { |path| path =~ %r{\A(ee/)?config/feature_flags/} }.map { |path| Found.new(path) }
end
+ # TODO: Move this to gitlab-dangerfiles helper
+ # https://gitlab.com/gitlab-org/ruby/gems/gitlab-dangerfiles/-/blob/master/lib/danger/plugins/internal/helper.rb
+ def stage_label
+ helper.mr_labels.find { |label| label.start_with?("devops::") }
+ end
+
class Found
ATTRIBUTES = %w[name introduced_by_url rollout_issue_url milestone type group default_enabled].freeze
diff --git a/tooling/danger/multiversion.rb b/tooling/danger/multiversion.rb
new file mode 100644
index 00000000000..612881aaa1a
--- /dev/null
+++ b/tooling/danger/multiversion.rb
@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Tooling
+ module Danger
+ module Multiversion
+ FRONTEND_REGEX = %r{\A((ee|jh)/)?app/assets/.*(\.(vue|js|graphql))\z}
+ GRAPHQL_BACKEND_REGEX = %r{\A((ee|jh)/)?app/graphql/}
+
+ def check!
+ return unless helper.ci?
+ return unless frontend_changed? && backend_changed?
+
+ markdown <<~MARKDOWN
+ ## Multiversion compatibility
+
+ This merge request updates GraphQL backend and frontend code.
+
+ To prevent an incident, ensure the updated frontend code is backwards compatible.
+
+ For more information, see the [multiversion compatibility documentation](https://docs.gitlab.com/ee/development/graphql_guide/reviewing.html#multiversion-compatibility).
+ MARKDOWN
+ end
+
+ private
+
+ def frontend_changed?
+ !git.modified_files.grep(FRONTEND_REGEX).empty? || !git.added_files.grep(FRONTEND_REGEX).empty?
+ end
+
+ def backend_changed?
+ !git.added_files.grep(GRAPHQL_BACKEND_REGEX).empty? || !git.modified_files.grep(GRAPHQL_BACKEND_REGEX).empty?
+ end
+ end
+ end
+end
diff --git a/tooling/danger/project_helper.rb b/tooling/danger/project_helper.rb
index 2a77ac337a2..5b5b998e3ea 100644
--- a/tooling/danger/project_helper.rb
+++ b/tooling/danger/project_helper.rb
@@ -26,11 +26,11 @@ module Tooling
%r{\Adoc/api/graphql/reference/} => [:docs, :backend],
%r{\Adoc/api/openapi/.*\.yaml\z} => [:docs, :backend],
- [%r{usage_data\.rb}, %r{^(\+|-).*\s+(count|distinct_count|estimate_batch_distinct_count)\(.*\)(.*)$}] => [:database, :backend, :product_intelligence],
+ [%r{usage_data\.rb}, %r{^(\+|-).*\s+(count|distinct_count|estimate_batch_distinct_count)\(.*\)(.*)$}] => [:database, :backend, :analytics_instrumentation],
%r{\A((ee|jh)/)?config/feature_flags/} => :feature_flag,
- %r{doc/api/usage_data.md} => [:product_intelligence],
+ %r{doc/api/usage_data.md} => [:analytics_instrumentation],
%r{\Adoc/.*(\.(md|png|gif|jpg|yml))\z} => :docs,
%r{\A(CONTRIBUTING|LICENSE|MAINTENANCE|PHILOSOPHY|PROCESS|README)(\.md)?\z} => :docs,
@@ -39,9 +39,9 @@ module Tooling
%r{\Adata/deprecations/} => :none,
%r{\Adata/removals/} => :none,
- %r{\A((ee|jh)/)?app/finders/(.+/)?integrations/} => [:integrations_be, :database, :backend],
- [%r{\A((ee|jh)/)?db/(geo/)?(migrate|post_migrate)/}, %r{(:integrations|:\w+_tracker_data)\b}] => [:integrations_be, :database, :migration],
- [%r{\A((ee|jh)/)?(app|lib)/.+\.rb}, %r{\b(Integrations::|\.execute_(integrations|hooks))\b}] => [:integrations_be, :backend],
+ %r{\A((ee|jh)/)?app/finders/(.+/)?integrations/} => [:import_integrate_be, :database, :backend],
+ [%r{\A((ee|jh)/)?db/(geo/)?(migrate|post_migrate)/}, %r{(:integrations|:\w+_tracker_data)\b}] => [:import_integrate_be, :database, :migration],
+ [%r{\A((ee|jh)/)?(app|lib)/.+\.rb}, %r{\b(Integrations::|\.execute_(integrations|hooks))\b}] => [:import_integrate_be, :backend],
%r{\A(
((ee|jh)/)?app/((?!.*clusters)(?!.*alert_management)(?!.*views)(?!.*assets).+/)?integration.+ |
((ee|jh)/)?app/((?!.*search).+/)?project_service.+ |
@@ -53,20 +53,20 @@ module Tooling
((ee|jh)/)?lib/(.+/)?.*integration.+ |
((ee|jh)/)?lib/(.+/)?api/v3/github\.rb |
((ee|jh)/)?lib/(.+/)?api/github/entities\.rb
- )\z}x => [:integrations_be, :backend],
+ )\z}x => [:import_integrate_be, :backend],
%r{\A(
((ee|jh)/)?app/(views|assets)/((?!.*clusters)(?!.*alerts_settings).+/)?integration.+ |
((ee|jh)/)?app/(views|assets)/(.+/)?jira_connect.+ |
((ee|jh)/)?app/(views|assets)/((?!.*filtered_search).+/)?hooks?.+
- )\z}x => [:integrations_fe, :frontend],
+ )\z}x => [:import_integrate_fe, :frontend],
%r{\A(
app/assets/javascripts/tracking/.*\.js |
spec/frontend/tracking/.*\.js |
spec/frontend/tracking_spec\.js
- )\z}x => [:frontend, :product_intelligence],
- [%r{\.(vue|js)\z}, %r{trackRedis}] => [:frontend, :product_intelligence],
+ )\z}x => [:frontend, :analytics_instrumentation],
+ [%r{\.(vue|js)\z}, %r{trackRedis}] => [:frontend, :analytics_instrumentation],
%r{\A((ee|jh)/)?app/assets/} => :frontend,
%r{\A((ee|jh)/)?app/views/.*\.svg} => :frontend,
%r{\A((ee|jh)/)?app/views/} => [:frontend, :backend],
@@ -132,11 +132,11 @@ module Tooling
%r{\A((ee|jh)/)?spec/support/shared_contexts/features/} => :test,
%r{\A((ee|jh)/)?spec/support/helpers/features/} => :test,
- %r{\A((spec/)?lib/generators/gitlab/usage_metric_)} => [:product_intelligence],
- %r{\A((ee|jh)/)?lib/gitlab/usage_data_counters/.*\.yml\z} => [:product_intelligence],
- %r{\A((ee|jh)/)?config/(events|metrics)/((.*\.yml)|(schema\.json))\z} => [:product_intelligence],
- %r{\A((ee|jh)/)?lib/gitlab/usage_data(_counters)?(/|\.rb)} => [:backend, :product_intelligence],
- %r{\A((ee|jh)/)?(spec/)?lib/gitlab/usage(/|\.rb)} => [:backend, :product_intelligence],
+ %r{\A((spec/)?lib/generators/gitlab/usage_metric_)} => [:analytics_instrumentation],
+ %r{\A((ee|jh)/)?lib/gitlab/usage_data_counters/.*\.yml\z} => [:analytics_instrumentation],
+ %r{\A((ee|jh)/)?config/(events|metrics)/((.*\.yml)|(schema\.json))\z} => [:analytics_instrumentation],
+ %r{\A((ee|jh)/)?lib/gitlab/usage_data(_counters)?(/|\.rb)} => [:backend, :analytics_instrumentation],
+ %r{\A((ee|jh)/)?(spec/)?lib/gitlab/usage(/|\.rb)} => [:backend, :analytics_instrumentation],
%r{\A(
lib/gitlab/tracking\.rb |
spec/lib/gitlab/tracking_spec\.rb |
@@ -146,11 +146,11 @@ module Tooling
(spec/)?lib/generators/gitlab/usage_metric_definition/redis_hll_generator(_spec)?\.rb |
lib/generators/rails/usage_metric_definition_generator\.rb |
spec/lib/generators/usage_metric_definition_generator_spec\.rb |
- generator_templates/usage_metric_definition/metric_definition\.yml)\z}x => [:backend, :product_intelligence],
- %r{gitlab/usage_data(_spec)?\.rb} => [:product_intelligence],
- [%r{\.haml\z}, %r{data: \{ track}] => [:product_intelligence],
- [%r{\.(rb|haml)\z}, %r{Gitlab::Tracking\.(event|enabled\?|options)$}] => [:product_intelligence],
- [%r{\.(vue|js)\z}, %r{(Tracking.event|/\btrack\(/|data-track-action)}] => [:product_intelligence],
+ generator_templates/usage_metric_definition/metric_definition\.yml)\z}x => [:backend, :analytics_instrumentation],
+ %r{gitlab/usage_data(_spec)?\.rb} => [:analytics_instrumentation],
+ [%r{\.haml\z}, %r{data: \{ track}] => [:analytics_instrumentation],
+ [%r{\.(rb|haml)\z}, %r{Gitlab::Tracking\.(event|enabled\?|options)$}] => [:analytics_instrumentation],
+ [%r{\.(vue|js)\z}, %r{(Tracking.event|/\btrack\(/|data-track-action)}] => [:analytics_instrumentation],
%r{\A((ee|jh)/)?app/(?!assets|views)[^/]+} => :backend,
%r{\A((ee|jh)/)?(bin|config|generator_templates|lib|rubocop)/} => :backend,
diff --git a/tooling/danger/sidekiq_args.rb b/tooling/danger/sidekiq_args.rb
new file mode 100644
index 00000000000..d06bb92ca6d
--- /dev/null
+++ b/tooling/danger/sidekiq_args.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Tooling
+ module Danger
+ module SidekiqArgs
+ include ::Danger::Helpers
+
+ WORKER_FILES_REGEX = 'app/workers'
+ EE_PREFIX = 'ee/'
+ DEF_PERFORM = "def perform"
+ DEF_PERFORM_REGEX = /[\s+-]*def perform\((.*)\)/
+ BEFORE_DEF_PERFORM_REGEX = /^[\s-]*def perform\b/
+ AFTER_DEF_PERFORM_REGEX = /^[\s+]*def perform\b/
+
+ SUGGEST_MR_COMMENT = <<~SUGGEST_COMMENT
+ Please follow the [sidekiq development guidelines](https://docs.gitlab.com/ee/development/sidekiq/compatibility_across_updates.html#changing-the-arguments-for-a-worker) when changing sidekiq worker arguments.
+ SUGGEST_COMMENT
+
+ def changed_worker_files(ee: :include)
+ changed_files = helper.all_changed_files
+ folder_prefix =
+ case ee
+ when :include
+ "(#{EE_PREFIX})?"
+ when :only
+ EE_PREFIX
+ when :exclude
+ nil
+ end
+
+ changed_files.grep(%r{\A#{folder_prefix}#{WORKER_FILES_REGEX}})
+ end
+
+ def args_changed?(diff)
+ # Find the "before" and "after" versions of the perform method definition
+ before_def_perform = diff.find { |line| BEFORE_DEF_PERFORM_REGEX.match?(line) }
+ after_def_perform = diff.find { |line| AFTER_DEF_PERFORM_REGEX.match?(line) }
+
+ # args are not changed if there is no before or after def perform method
+ return false unless before_def_perform && after_def_perform
+
+ # Extract the perform method arguments from the "before" and "after" versions
+ before_args, after_args = diff.flat_map { |line| line.scan(DEF_PERFORM_REGEX) }
+
+ before_args != after_args
+ end
+
+ def add_comment_for_matched_line(filename)
+ diff = helper.changed_lines(filename)
+ return unless args_changed?(diff)
+
+ file_lines = project_helper.file_lines(filename)
+
+ perform_method_line = file_lines.index { |line| line.include?(DEF_PERFORM) }
+ markdown(format(SUGGEST_MR_COMMENT), file: filename, line: perform_method_line.succ)
+ end
+ end
+ end
+end
diff --git a/tooling/danger/sidekiq_queues.rb b/tooling/danger/sidekiq_queues.rb
index ae32b128682..bd6480fcba6 100644
--- a/tooling/danger/sidekiq_queues.rb
+++ b/tooling/danger/sidekiq_queues.rb
@@ -14,7 +14,7 @@ module Tooling
def changed_queue_names
@changed_queue_names ||=
(new_queues.values_at(*old_queues.keys) - old_queues.values)
- .compact.map { |queue| queue[:name] }
+ .compact.map { |queue| queue[:name] } # rubocop:disable Rails/Pluck
end
private
diff --git a/tooling/danger/specs.rb b/tooling/danger/specs.rb
index f95a798d53e..68cab141656 100644
--- a/tooling/danger/specs.rb
+++ b/tooling/danger/specs.rb
@@ -1,55 +1,19 @@
# frozen_string_literal: true
-require_relative 'suggestor'
+Dir[File.expand_path('specs/*_suggestion.rb', __dir__)].each { |file| require file }
module Tooling
module Danger
module Specs
- include ::Tooling::Danger::Suggestor
-
SPEC_FILES_REGEX = 'spec/'
EE_PREFIX = 'ee/'
- PROJECT_FACTORIES = %w[
- :project
- :project_empty_repo
- :forked_project_with_submodules
- :project_with_design
+ SUGGESTIONS = [
+ FeatureCategorySuggestion,
+ MatchWithArraySuggestion,
+ ProjectFactorySuggestion
].freeze
- PROJECT_FACTORY_REGEX = /
- ^\+? # Start of the line, which may or may not have a `+`
- (?<head>\s*) # 0-many leading whitespace captured in a group named head
- let!? # Literal `let` which may or may not end in `!`
- (?<tail> # capture group named tail
- \([^)]+\) # Two parenthesis with any non-parenthesis characters between them
- \s*\{\s* # Opening curly brace surrounded by 0-many whitespace characters
- create\( # literal
- (?:#{PROJECT_FACTORIES.join('|')}) # Any of the project factory names
- \W # Non-word character, avoid matching factories like :project_authorization
- ) # end capture group named tail
- /x.freeze
-
- PROJECT_FACTORY_REPLACEMENT = '\k<head>let_it_be\k<tail>'
- PROJECT_FACTORY_SUGGESTION = <<~SUGGEST_COMMENT
- Project creations are very slow. Use `let_it_be`, `build` or `build_stubbed` if possible.
- See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#optimize-factory-usage)
- for background information and alternative options.
- SUGGEST_COMMENT
-
- MATCH_WITH_ARRAY_REGEX = /(?<to>to\(?\s*)(?<matcher>match|eq)(?<expectation>[( ]?\[(?=.*,)[^\]]+)/.freeze
- MATCH_WITH_ARRAY_REPLACEMENT = '\k<to>match_array\k<expectation>'
- MATCH_WITH_ARRAY_SUGGESTION = <<~SUGGEST_COMMENT
- If order of the result is not important, please consider using `match_array` to avoid flakiness.
- SUGGEST_COMMENT
-
- RSPEC_TOP_LEVEL_DESCRIBE_REGEX = /^\+.?RSpec\.describe(.+)/.freeze
- FEATURE_CATEGORY_SUGGESTION = <<~SUGGESTION_MARKDOWN
- Consider adding `feature_category: <feature_category_name>` for this example if it is not set already.
- See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#feature-category-metadata).
- SUGGESTION_MARKDOWN
- FEATURE_CATEGORY_KEYWORD = 'feature_category'
-
def changed_specs_files(ee: :include)
changed_files = helper.all_changed_files
folder_prefix =
@@ -65,43 +29,9 @@ module Tooling
changed_files.grep(%r{\A#{folder_prefix}#{SPEC_FILES_REGEX}})
end
- def add_suggestions_for_match_with_array(filename)
- add_suggestion(
- filename: filename,
- regex: MATCH_WITH_ARRAY_REGEX,
- replacement: MATCH_WITH_ARRAY_REPLACEMENT,
- comment_text: MATCH_WITH_ARRAY_SUGGESTION
- )
- end
-
- def add_suggestions_for_project_factory_usage(filename)
- add_suggestion(
- filename: filename,
- regex: PROJECT_FACTORY_REGEX,
- replacement: PROJECT_FACTORY_REPLACEMENT,
- comment_text: PROJECT_FACTORY_SUGGESTION
- )
- end
-
- def add_suggestions_for_feature_category(filename)
- file_lines = project_helper.file_lines(filename)
- changed_lines = helper.changed_lines(filename)
-
- changed_lines.each_with_index do |changed_line, i|
- next unless changed_line =~ RSPEC_TOP_LEVEL_DESCRIBE_REGEX
-
- next_line_in_file = file_lines[file_lines.find_index(changed_line.delete_prefix('+')) + 1]
-
- if changed_line.include?(FEATURE_CATEGORY_KEYWORD) || next_line_in_file.to_s.include?(FEATURE_CATEGORY_KEYWORD)
- next
- end
-
- line_number = file_lines.find_index(changed_line.delete_prefix('+'))
- next unless line_number
-
- suggested_line = file_lines[line_number]
-
- markdown(comment(FEATURE_CATEGORY_SUGGESTION, suggested_line), file: filename, line: line_number.succ)
+ def add_suggestions_for(filename)
+ SUGGESTIONS.each do |suggestion|
+ suggestion.new(filename, context: self).suggest
end
end
end
diff --git a/tooling/danger/specs/feature_category_suggestion.rb b/tooling/danger/specs/feature_category_suggestion.rb
new file mode 100644
index 00000000000..5acf73c8956
--- /dev/null
+++ b/tooling/danger/specs/feature_category_suggestion.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative '../suggestion'
+
+module Tooling
+ module Danger
+ module Specs
+ class FeatureCategorySuggestion < Suggestion
+ RSPEC_TOP_LEVEL_DESCRIBE_REGEX = /^\+.?RSpec\.describe(.+)/
+ SUGGESTION = <<~SUGGESTION_MARKDOWN
+ Consider adding `feature_category: <feature_category_name>` for this example if it is not set already.
+ See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#feature-category-metadata).
+ SUGGESTION_MARKDOWN
+ FEATURE_CATEGORY_KEYWORD = 'feature_category'
+
+ def suggest
+ file_lines = project_helper.file_lines(filename)
+ changed_lines = helper.changed_lines(filename)
+
+ changed_lines.each do |changed_line|
+ next unless changed_line =~ RSPEC_TOP_LEVEL_DESCRIBE_REGEX
+
+ line_number = file_lines.find_index(changed_line.delete_prefix('+'))
+ next unless line_number
+
+ # Get the top level RSpec.describe line and the next 5 lines
+ lines_to_check = file_lines[line_number, 5]
+ # Remove all the lines after the first one that ends in `do`
+ last_line_number_of_describe_declaration = lines_to_check.index { |line| line.end_with?(' do') }
+ lines_to_check = lines_to_check[0..last_line_number_of_describe_declaration]
+
+ next if lines_to_check.any? { |line| line.include?(FEATURE_CATEGORY_KEYWORD) }
+
+ suggested_line = file_lines[line_number]
+
+ markdown(comment(SUGGESTION, suggested_line), file: filename, line: line_number.succ)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/tooling/danger/specs/match_with_array_suggestion.rb b/tooling/danger/specs/match_with_array_suggestion.rb
new file mode 100644
index 00000000000..eb0f7a1a832
--- /dev/null
+++ b/tooling/danger/specs/match_with_array_suggestion.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative '../suggestion'
+
+module Tooling
+ module Danger
+ module Specs
+ class MatchWithArraySuggestion < Suggestion
+ MATCH = /(?<to>to\(?\s*)(?<matcher>match|eq)(?<expectation>[( ]?\[(?=.*,)[^\]]+)/
+ REPLACEMENT = '\k<to>match_array\k<expectation>'
+ SUGGESTION = <<~SUGGEST_COMMENT
+ If order of the result is not important, please consider using `match_array` to avoid flakiness.
+ SUGGEST_COMMENT
+ end
+ end
+ end
+end
diff --git a/tooling/danger/specs/project_factory_suggestion.rb b/tooling/danger/specs/project_factory_suggestion.rb
new file mode 100644
index 00000000000..78230db9585
--- /dev/null
+++ b/tooling/danger/specs/project_factory_suggestion.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative '../suggestion'
+
+module Tooling
+ module Danger
+ module Specs
+ class ProjectFactorySuggestion < Suggestion
+ PROJECT_FACTORIES = %w[
+ :project
+ :project_empty_repo
+ :forked_project_with_submodules
+ :project_with_design
+ ].freeze
+
+ MATCH = /
+ ^\+? # Start of the line, which may or may not have a `+`
+ (?<head>\s*) # 0-many leading whitespace captured in a group named head
+ let!? # Literal `let` which may or may not end in `!`
+ (?<tail> # capture group named tail
+ \([^)]+\) # Two parenthesis with any non-parenthesis characters between them
+ \s*\{\s* # Opening curly brace surrounded by 0-many whitespace characters
+ create\( # literal
+ (?:#{PROJECT_FACTORIES.join('|')}) # Any of the project factory names
+ \W # Non-word character, avoid matching factories like :project_badge
+ ) # end capture group named tail
+ /x
+
+ REPLACEMENT = '\k<head>let_it_be\k<tail>'
+ SUGGESTION = <<~SUGGEST_COMMENT
+ Project creations are very slow. Using `let_it_be`, `build` or `build_stubbed` can improve test performance.
+
+ Warning: `let_it_be` may not be suitable if your test modifies data as this could result in state leaks!
+
+ In those cases, please use `let_it_be_with_reload` or `let_it_be_with_refind` instead.
+
+ If your are unsure which is the right method to use,
+ please refer to [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#optimize-factory-usage)
+ for background information and alternative options for optimizing factory usage.
+
+ Feel free to ignore this comment if you know `let` or `let!` are the better options and/or worry about causing state leaks.
+ SUGGEST_COMMENT
+ end
+ end
+ end
+end
diff --git a/tooling/danger/stable_branch.rb b/tooling/danger/stable_branch.rb
index 9b467146096..bba198d1310 100644
--- a/tooling/danger/stable_branch.rb
+++ b/tooling/danger/stable_branch.rb
@@ -42,18 +42,18 @@ module Tooling
MSG
PIPELINE_EXPEDITE_ERROR_MESSAGE = <<~MSG
- ~"pipeline:expedite" is not allowed on stable branches because it causes the `e2e:package-and-test` job to be skipped.
+ ~"pipeline:expedite" is not allowed on stable branches because it causes the `e2e:package-and-test-ee` job to be skipped.
MSG
NEEDS_PACKAGE_AND_TEST_MESSAGE = <<~MSG
- The `e2e:package-and-test` job is not present, has been canceled, or needs to be automatically triggered.
+ The `e2e:package-and-test-ee` job is not present, has been canceled, or needs to be automatically triggered.
Please ensure the job is present in the latest pipeline, if necessary, retry the `danger-review` job.
- Read the "QA e2e:package-and-test" section for more details.
+ Read the "QA e2e:package-and-test-ee" section for more details.
MSG
WARN_PACKAGE_AND_TEST_MESSAGE = <<~MSG
- **The `e2e:package-and-test` job needs to succeed or have approval from a Software Engineer in Test.**
- Read the "QA e2e:package-and-test" section for more details.
+ **The `e2e:package-and-test-ee` job needs to succeed or have approval from a Software Engineer in Test.**
+ Read the "QA e2e:package-and-test-ee" section for more details.
MSG
# rubocop:disable Style/SignalException
@@ -74,7 +74,7 @@ module Tooling
if status.nil? || FAILING_PACKAGE_AND_TEST_STATUSES.include?(status) # rubocop:disable Style/GuardClause
fail NEEDS_PACKAGE_AND_TEST_MESSAGE
else
- warn WARN_PACKAGE_AND_TEST_MESSAGE unless status == 'success'
+ warn WARN_PACKAGE_AND_TEST_MESSAGE
end
end
# rubocop:enable Style/SignalException
@@ -85,12 +85,12 @@ module Tooling
!has_flaky_failure_label?
end
- private
-
def valid_stable_branch?
!!stable_target_branch && !helper.security_mr?
end
+ private
+
def package_and_test_bridge_and_pipeline_status
mr_head_pipeline_id = gitlab.mr_json.dig('head_pipeline', 'id')
return unless mr_head_pipeline_id
@@ -110,7 +110,7 @@ module Tooling
gitlab
.api
.pipeline_bridges(helper.mr_target_project_id, mr_head_pipeline_id)
- &.find { |bridge| bridge['name'] == 'e2e:package-and-test' }
+ &.find { |bridge| bridge['name'].include?('package-and-test-ee') }
end
def stable_target_branch
diff --git a/tooling/danger/suggestion.rb b/tooling/danger/suggestion.rb
new file mode 100644
index 00000000000..da3c6b0e76f
--- /dev/null
+++ b/tooling/danger/suggestion.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require_relative 'suggestor'
+
+module Tooling
+ module Danger
+ # A basic suggestion.
+ #
+ # A subclass needs to define the following constants:
+ # * MATCH (Regexp) - A Regexp to match file lines
+ # * REPLACEMENT (String) - A suggestion replacement text
+ # * SUGGESTION (String) - A suggestion text
+ #
+ # @see Suggestor
+ class Suggestion
+ extend Forwardable
+ include ::Tooling::Danger::Suggestor
+
+ def_delegators :@context, :helper, :project_helper, :markdown
+
+ attr_reader :filename
+
+ def initialize(filename, context:)
+ @filename = filename
+ @context = context
+ end
+
+ def suggest
+ add_suggestion(
+ filename: filename,
+ regex: self.class::MATCH,
+ replacement: self.class::REPLACEMENT,
+ comment_text: self.class::SUGGESTION
+ )
+ end
+ end
+ end
+end
diff --git a/tooling/docs/deprecation_handling.rb b/tooling/docs/deprecation_handling.rb
index bcdf73e0044..5996a0c89c1 100644
--- a/tooling/docs/deprecation_handling.rb
+++ b/tooling/docs/deprecation_handling.rb
@@ -1,4 +1,5 @@
# frozen_string_literal: true
+
require 'erb'
module Docs
@@ -7,11 +8,7 @@ module Docs
@type = type
@yaml_glob_path = Rails.root.join("data/#{type.pluralize}/**/*.yml")
@template_path = Rails.root.join("data/#{type.pluralize}/templates/_#{type}_template.md.erb")
- @milestone_key_name = if type == "deprecation"
- "announcement_milestone"
- else
- "removal_milestone"
- end
+ @milestone_key_name = "removal_milestone"
end
def render
diff --git a/tooling/graphql/docs/helper.rb b/tooling/graphql/docs/helper.rb
index f25e69a1e2f..bd8ff0cf862 100644
--- a/tooling/graphql/docs/helper.rb
+++ b/tooling/graphql/docs/helper.rb
@@ -56,7 +56,7 @@ module Tooling
<<-MD.strip_heredoc
---
stage: Manage
- group: Integrations
+ group: Import and Integrate
info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/product/ux/technical-writing/#assignments
---
diff --git a/tooling/lib/tooling/fast_quarantine.rb b/tooling/lib/tooling/fast_quarantine.rb
new file mode 100644
index 00000000000..a0dc8bc460b
--- /dev/null
+++ b/tooling/lib/tooling/fast_quarantine.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Tooling
+ class FastQuarantine
+ def initialize(fast_quarantine_path:)
+ warn "#{fast_quarantine_path} doesn't exist!" unless File.exist?(fast_quarantine_path.to_s)
+
+ @fast_quarantine_path = fast_quarantine_path
+ end
+
+ def identifiers
+ @identifiers ||= begin
+ quarantined_entity_identifiers = File.read(fast_quarantine_path).lines
+ quarantined_entity_identifiers.compact!
+ quarantined_entity_identifiers.map! do |quarantined_entity_identifier|
+ quarantined_entity_identifier.delete_prefix('./').strip
+ end
+ rescue => e # rubocop:disable Style/RescueStandardError
+ $stdout.puts e
+ []
+ end
+ end
+
+ def skip_example?(example)
+ identifiers.find do |quarantined_entity_identifier|
+ case quarantined_entity_identifier
+ when /^.+_spec\.rb\[[\d:]+\]$/ # example id, e.g. spec/tasks/gitlab/usage_data_rake_spec.rb[1:5:2:1]
+ example.id == "./#{quarantined_entity_identifier}"
+ when /^.+_spec\.rb:\d+$/ # file + line, e.g. spec/tasks/gitlab/usage_data_rake_spec.rb:42
+ fetch_metadata_from_ancestors(example, :location)
+ .any?("./#{quarantined_entity_identifier}")
+ when /^.+_spec\.rb$/ # whole file, e.g. ee/spec/features/boards/swimlanes/epics_swimlanes_sidebar_spec.rb
+ fetch_metadata_from_ancestors(example, :file_path)
+ .any?("./#{quarantined_entity_identifier}")
+ end
+ end
+ end
+
+ private
+
+ attr_reader :fast_quarantine_path
+
+ def fetch_metadata_from_ancestors(example, attribute)
+ metadata = [example.metadata[attribute]]
+ example_group = example.metadata[:example_group]
+
+ loop do
+ break if example_group.nil?
+
+ metadata << example_group[attribute]
+ example_group = example_group[:parent_example_group]
+ end
+
+ metadata
+ end
+ end
+end
diff --git a/tooling/lib/tooling/find_changes.rb b/tooling/lib/tooling/find_changes.rb
new file mode 100755
index 00000000000..c498c83d24b
--- /dev/null
+++ b/tooling/lib/tooling/find_changes.rb
@@ -0,0 +1,109 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'gitlab'
+require_relative 'helpers/predictive_tests_helper'
+
+module Tooling
+ class FindChanges
+ include Helpers::PredictiveTestsHelper
+
+ ALLOWED_FILE_TYPES = ['.js', '.vue', '.md', '.scss'].freeze
+
+ def initialize(
+ from:,
+ changed_files_pathname: nil,
+ predictive_tests_pathname: nil,
+ frontend_fixtures_mapping_pathname: nil
+ )
+
+ raise ArgumentError, ':from can only be :api or :changed_files' unless
+ %i[api changed_files].include?(from)
+
+ @gitlab_token = ENV['PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE'] || ''
+ @gitlab_endpoint = ENV['CI_API_V4_URL']
+ @mr_project_path = ENV['CI_MERGE_REQUEST_PROJECT_PATH']
+ @mr_iid = ENV['CI_MERGE_REQUEST_IID']
+ @changed_files_pathname = changed_files_pathname
+ @predictive_tests_pathname = predictive_tests_pathname
+ @frontend_fixtures_mapping_pathname = frontend_fixtures_mapping_pathname
+ @from = from
+ end
+
+ def execute
+ if changed_files_pathname.nil?
+ raise ArgumentError, "A path to the changed files file must be given as :changed_files_pathname"
+ end
+
+ case @from
+ when :api
+ write_array_to_file(changed_files_pathname, file_changes + frontend_fixture_files, append: false)
+ else
+ write_array_to_file(changed_files_pathname, frontend_fixture_files, append: true)
+ end
+ end
+
+ def only_allowed_files_changed
+ file_changes.any? && file_changes.all? { |file| ALLOWED_FILE_TYPES.include?(File.extname(file)) }
+ end
+
+ private
+
+ attr_reader :gitlab_token, :gitlab_endpoint, :mr_project_path,
+ :mr_iid, :changed_files_pathname, :predictive_tests_pathname, :frontend_fixtures_mapping_pathname
+
+ def gitlab
+ @gitlab ||= begin
+ Gitlab.configure do |config|
+ config.endpoint = gitlab_endpoint
+ config.private_token = gitlab_token
+ end
+
+ Gitlab
+ end
+ end
+
+ def add_frontend_fixture_files?
+ predictive_tests_pathname && frontend_fixtures_mapping_pathname
+ end
+
+ def frontend_fixture_files
+ # If we have a `test file -> JSON frontend fixture` mapping file, we add the files JSON frontend fixtures
+ # files to the list of changed files so that Jest can automatically run the dependent tests
+ # using --findRelatedTests flag.
+ empty = [].freeze
+
+ test_files.flat_map do |test_file|
+ frontend_fixtures_mapping[test_file] || empty
+ end
+ end
+
+ def file_changes
+ @file_changes ||=
+ case @from
+ when :api
+ mr_changes.changes.flat_map do |change|
+ change.to_h.values_at('old_path', 'new_path')
+ end.uniq
+ else
+ read_array_from_file(changed_files_pathname)
+ end
+ end
+
+ def mr_changes
+ @mr_changes ||= gitlab.merge_request_changes(mr_project_path, mr_iid)
+ end
+
+ def test_files
+ return [] if !predictive_tests_pathname || !File.exist?(predictive_tests_pathname)
+
+ read_array_from_file(predictive_tests_pathname)
+ end
+
+ def frontend_fixtures_mapping
+ return {} if !frontend_fixtures_mapping_pathname || !File.exist?(frontend_fixtures_mapping_pathname)
+
+ JSON.parse(File.read(frontend_fixtures_mapping_pathname)) # rubocop:disable Gitlab/Json
+ end
+ end
+end
diff --git a/tooling/lib/tooling/find_codeowners.rb b/tooling/lib/tooling/find_codeowners.rb
index cc37d4db1ec..e542ab9967c 100644
--- a/tooling/lib/tooling/find_codeowners.rb
+++ b/tooling/lib/tooling/find_codeowners.rb
@@ -48,11 +48,7 @@ module Tooling
def load_config
config_path = "#{__dir__}/../../config/CODEOWNERS.yml"
- if YAML.respond_to?(:safe_load_file) # Ruby 3.0+
- YAML.safe_load_file(config_path, symbolize_names: true)
- else
- YAML.safe_load(File.read(config_path), symbolize_names: true)
- end
+ YAML.safe_load_file(config_path, symbolize_names: true)
end
# Copied and modified from ee/lib/gitlab/code_owners/file.rb
diff --git a/tooling/lib/tooling/find_files_using_feature_flags.rb b/tooling/lib/tooling/find_files_using_feature_flags.rb
new file mode 100644
index 00000000000..27cace60408
--- /dev/null
+++ b/tooling/lib/tooling/find_files_using_feature_flags.rb
@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require 'test_file_finder'
+require_relative 'helpers/predictive_tests_helper'
+
+module Tooling
+ class FindFilesUsingFeatureFlags
+ include Helpers::PredictiveTestsHelper
+
+ def initialize(changed_files_pathname:, feature_flags_base_folder: 'config/feature_flags')
+ @changed_files_pathname = changed_files_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @feature_flags_base_folders = folders_for_available_editions(feature_flags_base_folder)
+ end
+
+ def execute
+ ff_union_regexp = Regexp.union(feature_flag_filenames)
+
+ files_using_modified_feature_flags = ruby_files.select do |ruby_file|
+ ruby_file if ff_union_regexp.match?(File.read(ruby_file))
+ end
+
+ write_array_to_file(changed_files_pathname, files_using_modified_feature_flags.uniq)
+ end
+
+ def filter_files
+ @_filter_files ||= changed_files.select do |filename|
+ filename.start_with?(*feature_flags_base_folders) &&
+ File.basename(filename).end_with?('.yml') &&
+ File.exist?(filename)
+ end
+ end
+
+ private
+
+ def feature_flag_filenames
+ filter_files.map do |feature_flag_pathname|
+ File.basename(feature_flag_pathname).delete_suffix('.yml')
+ end
+ end
+
+ def ruby_files
+ Dir["**/*.rb"].reject { |pathname| pathname.start_with?('vendor') }
+ end
+
+ attr_reader :changed_files, :changed_files_pathname, :feature_flags_base_folders
+ end
+end
diff --git a/tooling/lib/tooling/find_tests.rb b/tooling/lib/tooling/find_tests.rb
new file mode 100644
index 00000000000..bf7a608878b
--- /dev/null
+++ b/tooling/lib/tooling/find_tests.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'test_file_finder'
+require_relative 'helpers/predictive_tests_helper'
+
+module Tooling
+ class FindTests
+ include Helpers::PredictiveTestsHelper
+
+ def initialize(changed_files_pathname, predictive_tests_pathname)
+ @predictive_tests_pathname = predictive_tests_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ end
+
+ def execute
+ tff = TestFileFinder::FileFinder.new(paths: changed_files).tap do |file_finder|
+ file_finder.use TestFileFinder::MappingStrategies::PatternMatching.load('tests.yml')
+
+ if ENV['RSPEC_TESTS_MAPPING_ENABLED'] == 'true'
+ file_finder.use TestFileFinder::MappingStrategies::DirectMatching.load_json(ENV['RSPEC_TESTS_MAPPING_PATH'])
+ end
+ end
+
+ write_array_to_file(predictive_tests_pathname, tff.test_files.uniq)
+ end
+
+ private
+
+ attr_reader :changed_files, :matching_tests, :predictive_tests_pathname
+ end
+end
diff --git a/tooling/lib/tooling/gettext_extractor.rb b/tooling/lib/tooling/gettext_extractor.rb
new file mode 100644
index 00000000000..3cc05c80a2d
--- /dev/null
+++ b/tooling/lib/tooling/gettext_extractor.rb
@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require 'parallel'
+require 'gettext/po'
+require 'gettext/po_entry'
+require 'gettext/tools/xgettext'
+require 'gettext/tools/parser/erb'
+require 'gettext/tools/parser/ruby'
+require 'json'
+require 'open3'
+
+module Tooling
+ class GettextExtractor < GetText::Tools::XGetText
+ class HamlParser < GetText::RubyParser
+ require 'hamlit'
+ def parse_source(source)
+ super(Hamlit::Engine.new.call(source))
+ end
+ end
+
+ def initialize(
+ backend_glob: "{ee/,}{app,lib,config,locale}/**/*.{rb,erb,haml}",
+ glob_base: nil,
+ package_name: 'gitlab',
+ package_version: '1.0.0'
+ )
+ super()
+ @backend_glob = backend_glob
+ @package_name = package_name
+ @glob_base = glob_base || Dir.pwd
+ @package_version = package_version
+ # Ensure that the messages are ordered by id
+ @po_order = :msgid
+ @po_format_options = {
+ # No line breaks within a message
+ max_line_width: -1,
+ # Do not print references to files
+ include_reference_comment: false
+ }
+ end
+
+ def parse(_paths)
+ po = GetText::PO.new
+ parse_backend_files.each do |po_entry|
+ merge_po_entries(po, po_entry)
+ end
+ parse_frontend_files.each do |po_entry|
+ merge_po_entries(po, po_entry)
+ end
+ po
+ end
+
+ # Overrides method from GetText::Tools::XGetText
+ # This makes a method public and passes in an empty array of paths,
+ # as our overidden "parse" method needs no paths
+ def generate_pot
+ super([])
+ end
+
+ private
+
+ # Overrides method from GetText::Tools::XGetText
+ # in order to remove revision dates, as we check in our locale/gitlab.pot
+ def header_content
+ super.gsub(/^POT?-(?:Creation|Revision)-Date:.*\n/, '')
+ end
+
+ def merge_po_entries(po, po_entry)
+ existing_entry = po[po_entry.msgctxt, po_entry.msgid]
+ po_entry = existing_entry.merge(po_entry) if existing_entry
+
+ po[po_entry.msgctxt, po_entry.msgid] = po_entry
+ end
+
+ def parse_backend_file(path)
+ source = ::File.read(path)
+ # Do not bother parsing files not containing `_(`
+ # All of our translation helpers, _(, s_(), N_(), etc.
+ # contain it. So we can skip parsing files not containing it
+ return [] unless source.include?('_(')
+
+ case ::File.extname(path)
+ when '.rb'
+ GetText::RubyParser.new(path).parse_source(source)
+ when '.haml'
+ HamlParser.new(path).parse_source(source)
+ when '.erb'
+ GetText::ErbParser.new(path).parse
+ else
+ raise NotImplementedError
+ end
+ end
+
+ def parse_backend_files
+ files = Dir.glob(File.join(@glob_base, @backend_glob))
+ Parallel.flat_map(files) { |item| parse_backend_file(item) }
+ end
+
+ def parse_frontend_files
+ results, status = Open3.capture2('node scripts/frontend/extract_gettext_all.js --all')
+ raise StandardError, "Could not parse frontend files" unless status.success?
+
+ # rubocop:disable Gitlab/Json
+ JSON.parse(results)
+ .values
+ .flatten(1)
+ .collect { |entry| create_po_entry(*entry) }
+ # rubocop:enable Gitlab/Json
+ end
+ end
+end
diff --git a/tooling/lib/tooling/helpers/file_handler.rb b/tooling/lib/tooling/helpers/file_handler.rb
new file mode 100644
index 00000000000..88248e31df2
--- /dev/null
+++ b/tooling/lib/tooling/helpers/file_handler.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+
+module Tooling
+ module Helpers
+ module FileHandler
+ def read_array_from_file(file)
+ FileUtils.touch file
+
+ File.read(file).split(' ')
+ end
+
+ def write_array_to_file(file, content_array, append: true)
+ FileUtils.touch file
+
+ # We sort the array to make it easier to read the output file
+ content_array.sort!
+
+ output_content =
+ if append
+ [File.read(file), *content_array].join(' ').lstrip
+ else
+ content_array.join(' ')
+ end
+
+ File.write(file, output_content)
+ end
+ end
+ end
+end
diff --git a/tooling/lib/tooling/mappings/base.rb b/tooling/lib/tooling/helpers/predictive_tests_helper.rb
index 93d3a967114..b8e5a30024e 100644
--- a/tooling/lib/tooling/mappings/base.rb
+++ b/tooling/lib/tooling/helpers/predictive_tests_helper.rb
@@ -1,22 +1,13 @@
# frozen_string_literal: true
require_relative '../../../../lib/gitlab_edition'
+require_relative '../helpers/file_handler'
# Returns system specs files that are related to the JS files that were changed in the MR.
module Tooling
- module Mappings
- class Base
- # Input: A list of space-separated files
- # Output: A list of space-separated specs files (JS, Ruby, ...)
- def execute(changed_files)
- raise "Not Implemented"
- end
-
- # Input: A list of space-separated files
- # Output: array/hash of files
- def filter_files(changed_files)
- raise "Not Implemented"
- end
+ module Helpers
+ module PredictiveTestsHelper
+ include FileHandler
# Input: A folder
# Output: An array of folders, each prefixed with a GitLab edition
diff --git a/tooling/lib/tooling/kubernetes_client.rb b/tooling/lib/tooling/kubernetes_client.rb
index ab914db5777..5579f130a84 100644
--- a/tooling/lib/tooling/kubernetes_client.rb
+++ b/tooling/lib/tooling/kubernetes_client.rb
@@ -6,72 +6,23 @@ require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen)
module Tooling
class KubernetesClient
- RESOURCE_LIST = 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd'
K8S_ALLOWED_NAMESPACES_REGEX = /^review-(?!apps).+/.freeze
CommandFailedError = Class.new(StandardError)
- attr_reader :namespace
+ def cleanup_namespaces_by_created_at(created_before:)
+ stale_namespaces = namespaces_created_before(created_before: created_before)
- def initialize(namespace:)
- @namespace = namespace
- end
-
- def cleanup_by_release(release_name:, wait: true)
- delete_by_selector(release_name: release_name, wait: wait)
- delete_by_matching_name(release_name: release_name)
- end
-
- def cleanup_by_created_at(resource_type:, created_before:, wait: true)
- resource_names = resource_names_created_before(resource_type: resource_type, created_before: created_before)
- return if resource_names.empty?
-
- delete_by_exact_names(resource_type: resource_type, resource_names: resource_names, wait: wait)
- end
-
- def cleanup_review_app_namespaces(created_before:, wait: true)
- namespaces = review_app_namespaces_created_before(created_before: created_before)
- return if namespaces.empty?
-
- delete_namespaces_by_exact_names(resource_names: namespaces, wait: wait)
- end
-
- private
-
- def delete_by_selector(release_name:, wait:)
- selector = case release_name
- when String
- %(-l release="#{release_name}")
- when Array
- %(-l 'release in (#{release_name.join(', ')})')
- else
- raise ArgumentError, 'release_name must be a string or an array'
- end
-
- command = [
- 'delete',
- RESOURCE_LIST,
- %(--namespace "#{namespace}"),
- '--now',
- '--ignore-not-found',
- %(--wait=#{wait}),
- selector
- ]
+ # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby.
+ review_apps_stale_namespaces = stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) }
+ return if review_apps_stale_namespaces.empty?
- run_command(command)
+ delete_namespaces(review_apps_stale_namespaces)
end
- def delete_by_exact_names(resource_names:, wait:, resource_type: nil)
- command = [
- 'delete',
- resource_type,
- %(--namespace "#{namespace}"),
- '--now',
- '--ignore-not-found',
- %(--wait=#{wait}),
- resource_names.join(' ')
- ]
+ def delete_namespaces(namespaces)
+ return if namespaces.any? { |ns| !K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) }
- run_command(command)
+ run_command("kubectl delete namespace --now --ignore-not-found #{namespaces.join(' ')}")
end
def delete_namespaces_by_exact_names(resource_names:, wait:)
@@ -87,87 +38,29 @@ module Tooling
run_command(command)
end
- def delete_by_matching_name(release_name:)
- resource_names = raw_resource_names
- command = [
- 'delete',
- %(--namespace "#{namespace}"),
- '--ignore-not-found'
- ]
-
- Array(release_name).each do |release|
- resource_names
- .select { |resource_name| resource_name.include?(release) }
- .each { |matching_resource| run_command(command + [matching_resource]) }
- end
- end
-
- def raw_resource_names
- command = [
- 'get',
- RESOURCE_LIST,
- %(--namespace "#{namespace}"),
- '-o name'
- ]
- run_command(command).lines.map(&:strip)
- end
-
- def resource_names_created_before(resource_type:, created_before:)
- command = [
- 'get',
- resource_type,
- %(--namespace "#{namespace}"),
- "--sort-by='{.metadata.creationTimestamp}'",
- '-o json'
- ]
+ def namespaces_created_before(created_before:)
+ response = run_command("kubectl get namespace --all-namespaces --sort-by='{.metadata.creationTimestamp}' -o json")
- response = run_command(command)
-
- resources_created_before_date(response, created_before)
- end
-
- def review_app_namespaces_created_before(created_before:)
- command = [
- 'get',
- 'namespace',
- "--sort-by='{.metadata.creationTimestamp}'",
- '-o json'
- ]
-
- response = run_command(command)
-
- stale_namespaces = resources_created_before_date(response, created_before)
-
- # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby.
- stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) }
- end
-
- def resources_created_before_date(response, date)
items = JSON.parse(response)['items'] # rubocop:disable Gitlab/Json
-
- items.each_with_object([]) do |item, result|
+ items.filter_map do |item|
item_created_at = Time.parse(item.dig('metadata', 'creationTimestamp'))
- if item_created_at < date
- resource_name = item.dig('metadata', 'name')
- result << resource_name
- end
+ item.dig('metadata', 'name') if item_created_at < created_before
end
rescue ::JSON::ParserError => ex
- puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output
+ puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}"
[]
end
def run_command(command)
- final_command = ['kubectl', *command.compact].join(' ')
- puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output
+ puts "Running command: `#{command}`"
- result = Gitlab::Popen.popen_with_detail([final_command])
+ result = Gitlab::Popen.popen_with_detail([command])
if result.status.success?
result.stdout.chomp.freeze
else
- raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
+ raise CommandFailedError, "The `#{command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}"
end
end
end
diff --git a/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb b/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb
new file mode 100644
index 00000000000..80aa99efc96
--- /dev/null
+++ b/tooling/lib/tooling/mappings/graphql_base_type_mappings.rb
@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require 'active_support/inflector'
+
+require_relative '../helpers/predictive_tests_helper'
+require_relative '../../../../lib/gitlab_edition'
+
+# If a GraphQL type class changed, we try to identify the other GraphQL types that potentially include this type.
+module Tooling
+ module Mappings
+ class GraphqlBaseTypeMappings
+ include Helpers::PredictiveTestsHelper
+
+ # Checks for the implements keyword, and graphql_base_types the class name
+ GRAPHQL_IMPLEMENTS_REGEXP = /implements[( ]([\w:]+)[)]?$/
+
+ # GraphQL types are a bit scattered in the codebase based on the edition.
+ #
+ # Also, a higher edition is able to include lower editions.
+ # e.g. EE can include FOSS GraphQL types, and JH can include all GraphQL types
+ GRAPHQL_TYPES_FOLDERS_FOSS = ['app/graphql/types'].freeze
+ GRAPHQL_TYPES_FOLDERS_EE = GRAPHQL_TYPES_FOLDERS_FOSS + ['ee/app/graphql/types', 'ee/app/graphql/ee/types']
+ GRAPHQL_TYPES_FOLDERS_JH = GRAPHQL_TYPES_FOLDERS_EE + ['jh/app/graphql/types', 'jh/app/graphql/jh/types']
+ GRAPHQL_TYPES_FOLDERS = {
+ nil => GRAPHQL_TYPES_FOLDERS_FOSS,
+ 'ee' => GRAPHQL_TYPES_FOLDERS_EE,
+ 'jh' => GRAPHQL_TYPES_FOLDERS_JH
+ }.freeze
+
+ def initialize(changed_files_pathname, predictive_tests_pathname)
+ @predictive_tests_pathname = predictive_tests_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ end
+
+ def execute
+ # We go through the available editions when searching for base types
+ #
+ # `nil` is the FOSS edition
+ matching_graphql_tests = ([nil] + ::GitlabEdition.extensions).flat_map do |edition|
+ hierarchy = types_hierarchies[edition]
+
+ filter_files.flat_map do |graphql_file|
+ children_types = hierarchy[filename_to_class_name(graphql_file)]
+ next if children_types.empty?
+
+ # We find the specs for the children GraphQL types that are implementing the current GraphQL Type
+ children_types.map { |filename| filename_to_spec_filename(filename) }
+ end
+ end.compact.uniq
+
+ write_array_to_file(predictive_tests_pathname, matching_graphql_tests)
+ end
+
+ def filter_files
+ changed_files.select do |filename|
+ filename.start_with?(*GRAPHQL_TYPES_FOLDERS.values.flatten.uniq) &&
+ filename.end_with?('.rb') &&
+ File.exist?(filename)
+ end
+ end
+
+ # Regroup all GraphQL types (by edition) that are implementing another GraphQL type.
+ #
+ # The key is the type that is being implemented (e.g. NoteableInterface, TodoableInterface below)
+ # The value is an array of GraphQL type files that are implementing those types.
+ #
+ # Example output:
+ #
+ # {
+ # nil => {
+ # "NoteableInterface" => [
+ # "app/graphql/types/alert_management/alert_type.rb",
+ # "app/graphql/types/design_management/design_type.rb"
+ # , "TodoableInterface" => [...]
+ # },
+ # "ee" => {
+ # "NoteableInterface" => [
+ # "app/graphql/types/alert_management/alert_type.rb",
+ # "app/graphql/types/design_management/design_type.rb",
+ # "ee/app/graphql/types/epic_type.rb"],
+ # "TodoableInterface"=> [...]
+ # }
+ # }
+ def types_hierarchies
+ return @types_hierarchies if @types_hierarchies
+
+ @types_hierarchies = {}
+ GRAPHQL_TYPES_FOLDERS.each_key do |edition|
+ @types_hierarchies[edition] = Hash.new { |h, k| h[k] = [] }
+
+ graphql_files_for_edition_glob = File.join("{#{GRAPHQL_TYPES_FOLDERS[edition].join(',')}}", '**', '*.rb')
+ Dir[graphql_files_for_edition_glob].each do |graphql_file|
+ graphql_base_types = File.read(graphql_file).scan(GRAPHQL_IMPLEMENTS_REGEXP)
+ next if graphql_base_types.empty?
+
+ graphql_base_classes = graphql_base_types.flatten.map { |class_name| class_name.split('::').last }
+ graphql_base_classes.each do |graphql_base_class|
+ @types_hierarchies[edition][graphql_base_class] += [graphql_file]
+ end
+ end
+ end
+
+ @types_hierarchies
+ end
+
+ def filename_to_class_name(filename)
+ File.basename(filename, '.*').camelize
+ end
+
+ def filename_to_spec_filename(filename)
+ spec_file = filename.sub('app', 'spec').sub('.rb', '_spec.rb')
+
+ return spec_file if File.exist?(spec_file)
+ end
+
+ private
+
+ attr_reader :changed_files, :predictive_tests_pathname
+ end
+ end
+end
diff --git a/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb
index 365e466011b..bc2cd259fdc 100644
--- a/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb
+++ b/tooling/lib/tooling/mappings/js_to_system_specs_mappings.rb
@@ -2,40 +2,49 @@
require 'active_support/inflector'
-require_relative 'base'
+require_relative '../helpers/predictive_tests_helper'
require_relative '../../../../lib/gitlab_edition'
# Returns system specs files that are related to the JS files that were changed in the MR.
module Tooling
module Mappings
- class JsToSystemSpecsMappings < Base
- def initialize(js_base_folder: 'app/assets/javascripts', system_specs_base_folder: 'spec/features')
- @js_base_folder = js_base_folder
- @js_base_folders = folders_for_available_editions(js_base_folder)
- @system_specs_base_folder = system_specs_base_folder
+ class JsToSystemSpecsMappings
+ include Helpers::PredictiveTestsHelper
+
+ def initialize(
+ changed_files_pathname, predictive_tests_pathname,
+ js_base_folder: 'app/assets/javascripts', system_specs_base_folder: 'spec/features')
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @predictive_tests_pathname = predictive_tests_pathname
+ @js_base_folder = js_base_folder
+ @js_base_folders = folders_for_available_editions(js_base_folder)
+ @system_specs_base_folder = system_specs_base_folder
# Cannot be extracted to a constant, as it depends on a variable
@first_js_folder_extract_regexp = %r{
(?:.*/)? # Skips the GitLab edition (e.g. ee/, jh/)
#{@js_base_folder}/ # Most likely app/assets/javascripts/
+ (?:pages/)? # If under a pages folder, we capture the following folder
([\w-]*) # Captures the first folder
}x
end
- def execute(changed_files)
- filter_files(changed_files).flat_map do |edition, js_files|
+ def execute
+ matching_system_tests = filter_files.flat_map do |edition, js_files|
js_keywords_regexp = Regexp.union(construct_js_keywords(js_files))
system_specs_for_edition(edition).select do |system_spec_file|
system_spec_file if js_keywords_regexp.match?(system_spec_file)
end
end
+
+ write_array_to_file(predictive_tests_pathname, matching_system_tests)
end
# Keep the files that are in the @js_base_folders folders
#
# Returns a hash, where the key is the GitLab edition, and the values the JS specs
- def filter_files(changed_files)
+ def filter_files
selected_files = changed_files.select do |filename|
filename.start_with?(*@js_base_folders) && File.exist?(filename)
end
@@ -54,8 +63,12 @@ module Tooling
def system_specs_for_edition(edition)
all_files_in_folders_glob = File.join(@system_specs_base_folder, '**', '*')
all_files_in_folders_glob = File.join(edition, all_files_in_folders_glob) if edition
- Dir[all_files_in_folders_glob].select { |f| File.file?(f) }
+ Dir[all_files_in_folders_glob].select { |f| File.file?(f) && f.end_with?('_spec.rb') }
end
+
+ private
+
+ attr_reader :changed_files, :predictive_tests_pathname
end
end
end
diff --git a/tooling/lib/tooling/mappings/partial_to_views_mappings.rb b/tooling/lib/tooling/mappings/partial_to_views_mappings.rb
new file mode 100644
index 00000000000..931cacea77f
--- /dev/null
+++ b/tooling/lib/tooling/mappings/partial_to_views_mappings.rb
@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require_relative '../helpers/predictive_tests_helper'
+require_relative '../../../../lib/gitlab_edition'
+
+# Returns view files that include the potential rails partials from the changed files passed as input.
+module Tooling
+ module Mappings
+ class PartialToViewsMappings
+ include Helpers::PredictiveTestsHelper
+
+ def initialize(changed_files_pathname, views_with_partials_pathname, view_base_folder: 'app/views')
+ @views_with_partials_pathname = views_with_partials_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @view_base_folders = folders_for_available_editions(view_base_folder)
+ end
+
+ def execute
+ views_including_modified_partials = []
+
+ views_globs = view_base_folders.map { |view_base_folder| "#{view_base_folder}/**/*.html.haml" }
+ Dir[*views_globs].each do |view_file|
+ included_partial_names = find_pattern_in_file(view_file, partials_keywords_regexp)
+ next if included_partial_names.empty?
+
+ included_partial_names.each do |included_partial_name|
+ if view_includes_modified_partial?(view_file, included_partial_name)
+ views_including_modified_partials << view_file
+ end
+ end
+ end
+
+ write_array_to_file(views_with_partials_pathname, views_including_modified_partials)
+ end
+
+ def filter_files
+ @_filter_files ||= changed_files.select do |filename|
+ filename.start_with?(*view_base_folders) &&
+ File.basename(filename).start_with?('_') &&
+ File.basename(filename).end_with?('.html.haml') &&
+ File.exist?(filename)
+ end
+ end
+
+ def partials_keywords_regexp
+ partial_keywords = filter_files.map do |partial_filename|
+ extract_partial_keyword(partial_filename)
+ end
+
+ partial_regexps = partial_keywords.map do |keyword|
+ %r{(?:render|render_if_exists)(?: |\()(?:partial: ?)?['"]([\w\-_/]*#{keyword})['"]}
+ end
+
+ Regexp.union(partial_regexps)
+ end
+
+ # e.g. if app/views/clusters/clusters/_sidebar.html.haml was modified, the partial keyword is `sidebar`.
+ def extract_partial_keyword(partial_filename)
+ File.basename(partial_filename).delete_prefix('_').delete_suffix('.html.haml')
+ end
+
+ # Why do we need this method?
+ #
+ # Assume app/views/clusters/clusters/_sidebar.html.haml was modified in the MR.
+ #
+ # Suppose now you find = render 'sidebar' in a view. Is this view including the sidebar partial
+ # that was modified, or another partial called "_sidebar.html.haml" somewhere else?
+ def view_includes_modified_partial?(view_file, included_partial_name)
+ view_file_parent_folder = File.dirname(view_file)
+ included_partial_filename = reconstruct_partial_filename(included_partial_name)
+ included_partial_relative_path = File.join(view_file_parent_folder, included_partial_filename)
+
+ # We do this because in render (or render_if_exists)
+ # apparently looks for partials in other GitLab editions
+ #
+ # Example:
+ #
+ # ee/app/views/events/_epics_filter.html.haml is used in app/views/shared/_event_filter.html.haml
+ # with render_if_exists 'events/epics_filter'
+ included_partial_absolute_paths = view_base_folders.map do |view_base_folder|
+ File.join(view_base_folder, included_partial_filename)
+ end
+
+ filter_files.include?(included_partial_relative_path) ||
+ (filter_files & included_partial_absolute_paths).any?
+ end
+
+ def reconstruct_partial_filename(partial_name)
+ partial_path = partial_name.split('/')[..-2]
+ partial_filename = partial_name.split('/').last
+ full_partial_filename = "_#{partial_filename}.html.haml"
+
+ return full_partial_filename if partial_path.empty?
+
+ File.join(partial_path.join('/'), full_partial_filename)
+ end
+
+ def find_pattern_in_file(file, pattern)
+ File.read(file).scan(pattern).flatten.compact.uniq
+ end
+
+ private
+
+ attr_reader :changed_files, :views_with_partials_pathname, :view_base_folders
+ end
+ end
+end
diff --git a/tooling/lib/tooling/mappings/view_to_js_mappings.rb b/tooling/lib/tooling/mappings/view_to_js_mappings.rb
index db80eb9bfe8..b78c354f9d2 100644
--- a/tooling/lib/tooling/mappings/view_to_js_mappings.rb
+++ b/tooling/lib/tooling/mappings/view_to_js_mappings.rb
@@ -1,47 +1,53 @@
# frozen_string_literal: true
-require_relative 'base'
+require_relative '../helpers/predictive_tests_helper'
require_relative '../../../../lib/gitlab_edition'
# Returns JS files that are related to the Rails views files that were changed in the MR.
module Tooling
module Mappings
- class ViewToJsMappings < Base
+ class ViewToJsMappings
+ include Helpers::PredictiveTestsHelper
+
# The HTML attribute value pattern we're looking for to match an HTML file to a JS file.
HTML_ATTRIBUTE_VALUE_REGEXP = /js-[-\w]+/.freeze
# Search for Rails partials included in an HTML file
- RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exist)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze
+ RAILS_PARTIAL_INVOCATION_REGEXP = %r{(?:render|render_if_exists)(?: |\()(?:partial: ?)?['"]([\w/-]+)['"]}.freeze
- def initialize(view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts')
- @view_base_folders = folders_for_available_editions(view_base_folder)
- @js_base_folders = folders_for_available_editions(js_base_folder)
+ def initialize(
+ changed_files_pathname, predictive_tests_pathname,
+ view_base_folder: 'app/views', js_base_folder: 'app/assets/javascripts')
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @predictive_tests_pathname = predictive_tests_pathname
+ @view_base_folders = folders_for_available_editions(view_base_folder)
+ @js_base_folders = folders_for_available_editions(js_base_folder)
end
- def execute(changed_files)
- changed_view_files = filter_files(changed_files)
-
- partials = changed_view_files.flat_map do |file|
+ def execute
+ partials = filter_files.flat_map do |file|
find_partials(file)
end
- files_to_scan = changed_view_files + partials
+ files_to_scan = filter_files + partials
js_tags = files_to_scan.flat_map do |file|
find_pattern_in_file(file, HTML_ATTRIBUTE_VALUE_REGEXP)
end
js_tags_regexp = Regexp.union(js_tags)
- @js_base_folders.flat_map do |js_base_folder|
+ matching_js_files = @js_base_folders.flat_map do |js_base_folder|
Dir["#{js_base_folder}/**/*.{js,vue}"].select do |js_file|
file_content = File.read(js_file)
js_tags_regexp.match?(file_content)
end
end
+
+ write_array_to_file(predictive_tests_pathname, matching_js_files)
end
# Keep the files that are in the @view_base_folders folder
- def filter_files(changed_files)
- changed_files.select do |filename|
+ def filter_files
+ @_filter_files ||= changed_files.select do |filename|
filename.start_with?(*@view_base_folders) &&
File.exist?(filename)
end
@@ -69,6 +75,10 @@ module Tooling
def find_pattern_in_file(file, pattern)
File.read(file).scan(pattern).flatten.uniq
end
+
+ private
+
+ attr_reader :changed_files, :predictive_tests_pathname
end
end
end
diff --git a/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb b/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb
new file mode 100644
index 00000000000..1542c817745
--- /dev/null
+++ b/tooling/lib/tooling/mappings/view_to_system_specs_mappings.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require_relative '../helpers/predictive_tests_helper'
+require_relative '../../../../lib/gitlab_edition'
+
+# Returns system specs files that are related to the Rails views files that were changed in the MR.
+module Tooling
+ module Mappings
+ class ViewToSystemSpecsMappings
+ include Helpers::PredictiveTestsHelper
+
+ def initialize(changed_files_pathname, predictive_tests_pathname, view_base_folder: 'app/views')
+ @predictive_tests_pathname = predictive_tests_pathname
+ @changed_files = read_array_from_file(changed_files_pathname)
+ @view_base_folders = folders_for_available_editions(view_base_folder)
+ end
+
+ def execute
+ found_system_specs = []
+
+ filter_files.each do |modified_view_file|
+ system_specs_exact_match = find_system_specs_exact_match(modified_view_file)
+ if system_specs_exact_match
+ found_system_specs << system_specs_exact_match
+ next
+ else
+ system_specs_parent_folder_match = find_system_specs_parent_folder_match(modified_view_file)
+ found_system_specs += system_specs_parent_folder_match unless system_specs_parent_folder_match.empty?
+ end
+ end
+
+ write_array_to_file(predictive_tests_pathname, found_system_specs.compact.uniq.sort)
+ end
+
+ private
+
+ attr_reader :changed_files, :predictive_tests_pathname, :view_base_folders
+
+ # Keep the views files that are in the @view_base_folders folder
+ def filter_files
+ @_filter_files ||= changed_files.select do |filename|
+ filename.start_with?(*view_base_folders) &&
+ File.basename(filename).end_with?('.html.haml') &&
+ File.exist?(filename)
+ end
+ end
+
+ def find_system_specs_exact_match(view_file)
+ potential_spec_file = to_feature_spec_folder(view_file).sub('.html.haml', '_spec.rb')
+
+ potential_spec_file if File.exist?(potential_spec_file)
+ end
+
+ def find_system_specs_parent_folder_match(view_file)
+ parent_system_specs_folder = File.dirname(to_feature_spec_folder(view_file))
+
+ Dir["#{parent_system_specs_folder}/**/*_spec.rb"]
+ end
+
+ # e.g. go from app/views/groups/merge_requests.html.haml to spec/features/groups/merge_requests.html.haml
+ def to_feature_spec_folder(view_file)
+ view_file.sub(%r{(ee/|jh/)?app/views}, '\1spec/features')
+ end
+ end
+ end
+end
diff --git a/tooling/lib/tooling/parallel_rspec_runner.rb b/tooling/lib/tooling/parallel_rspec_runner.rb
index b1ddc91e831..834d9ec23a7 100644
--- a/tooling/lib/tooling/parallel_rspec_runner.rb
+++ b/tooling/lib/tooling/parallel_rspec_runner.rb
@@ -1,6 +1,60 @@
# frozen_string_literal: true
require 'knapsack'
+require 'fileutils'
+
+module Knapsack
+ module Distributors
+ class BaseDistributor
+ # Refine https://github.com/KnapsackPro/knapsack/blob/v1.21.1/lib/knapsack/distributors/base_distributor.rb
+ # to take in account the additional filtering we do for predictive jobs.
+ module BaseDistributorWithTestFiltering
+ attr_reader :filter_tests
+
+ def initialize(args = {})
+ super
+
+ @filter_tests = args[:filter_tests]
+ end
+
+ def all_tests
+ @all_tests ||= begin
+ pattern_tests = Dir.glob(test_file_pattern).uniq
+
+ if filter_tests.empty?
+ pattern_tests
+ else
+ pattern_tests & filter_tests
+ end
+ end.sort
+ end
+ end
+
+ prepend BaseDistributorWithTestFiltering
+ end
+ end
+
+ class AllocatorBuilder
+ # Refine https://github.com/KnapsackPro/knapsack/blob/v1.21.1/lib/knapsack/allocator_builder.rb
+ # to take in account the additional filtering we do for predictive jobs.
+ module AllocatorBuilderWithTestFiltering
+ attr_accessor :filter_tests
+
+ def allocator
+ Knapsack::Allocator.new({
+ report: Knapsack.report.open,
+ test_file_pattern: test_file_pattern,
+ ci_node_total: Knapsack::Config::Env.ci_node_total,
+ ci_node_index: Knapsack::Config::Env.ci_node_index,
+ # Additional argument
+ filter_tests: filter_tests
+ })
+ end
+ end
+
+ prepend AllocatorBuilderWithTestFiltering
+ end
+end
# A custom parallel rspec runner based on Knapsack runner
# which takes in additional option for a file containing
@@ -13,32 +67,26 @@ require 'knapsack'
# would be executed in the CI node.
#
# Reference:
-# https://github.com/ArturT/knapsack/blob/v1.20.0/lib/knapsack/runners/rspec_runner.rb
+# https://github.com/ArturT/knapsack/blob/v1.21.1/lib/knapsack/runners/rspec_runner.rb
module Tooling
class ParallelRSpecRunner
def self.run(rspec_args: nil, filter_tests_file: nil)
new(rspec_args: rspec_args, filter_tests_file: filter_tests_file).run
end
- def initialize(allocator: knapsack_allocator, filter_tests_file: nil, rspec_args: nil)
- @allocator = allocator
+ def initialize(filter_tests_file: nil, rspec_args: nil)
@filter_tests_file = filter_tests_file
@rspec_args = rspec_args&.split(' ') || []
end
def run
- Knapsack.logger.info
- Knapsack.logger.info 'Knapsack node specs:'
- Knapsack.logger.info node_tests
- Knapsack.logger.info
- Knapsack.logger.info 'Filter specs:'
- Knapsack.logger.info filter_tests
- Knapsack.logger.info
- Knapsack.logger.info 'Running specs:'
- Knapsack.logger.info tests_to_run
- Knapsack.logger.info
-
- if tests_to_run.empty?
+ if ENV['KNAPSACK_RSPEC_SUITE_REPORT_PATH']
+ knapsack_dir = File.dirname(ENV['KNAPSACK_RSPEC_SUITE_REPORT_PATH'])
+ FileUtils.mkdir_p(knapsack_dir)
+ File.write(File.join(knapsack_dir, 'node_specs.txt'), node_tests.join("\n"))
+ end
+
+ if node_tests.empty?
Knapsack.logger.info 'No tests to run on this node, exiting.'
return
end
@@ -50,26 +98,16 @@ module Tooling
private
- attr_reader :allocator, :filter_tests_file, :rspec_args
+ attr_reader :filter_tests_file, :rspec_args
def rspec_command
%w[bundle exec rspec].tap do |cmd|
cmd.push(*rspec_args)
- cmd.push('--default-path', allocator.test_dir)
cmd.push('--')
- cmd.push(*tests_to_run)
+ cmd.push(*node_tests)
end
end
- def tests_to_run
- if filter_tests.empty?
- Knapsack.logger.info 'Running all node tests without filter'
- return node_tests
- end
-
- @tests_to_run ||= node_tests & filter_tests
- end
-
def node_tests
allocator.node_tests
end
@@ -85,8 +123,11 @@ module Tooling
File.read(filter_tests_file).split(' ')
end
- def knapsack_allocator
- Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator
+ def allocator
+ @allocator ||=
+ Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).tap do |builder|
+ builder.filter_tests = filter_tests
+ end.allocator
end
end
end
diff --git a/tooling/lib/tooling/predictive_tests.rb b/tooling/lib/tooling/predictive_tests.rb
new file mode 100644
index 00000000000..1ad63e111e3
--- /dev/null
+++ b/tooling/lib/tooling/predictive_tests.rb
@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require_relative 'find_changes'
+require_relative 'find_tests'
+require_relative 'find_files_using_feature_flags'
+require_relative 'mappings/graphql_base_type_mappings'
+require_relative 'mappings/js_to_system_specs_mappings'
+require_relative 'mappings/partial_to_views_mappings'
+require_relative 'mappings/view_to_js_mappings'
+require_relative 'mappings/view_to_system_specs_mappings'
+
+module Tooling
+ class PredictiveTests
+ REQUIRED_ENV_VARIABLES = %w[
+ RSPEC_CHANGED_FILES_PATH
+ RSPEC_MATCHING_TESTS_PATH
+ RSPEC_VIEWS_INCLUDING_PARTIALS_PATH
+ FRONTEND_FIXTURES_MAPPING_PATH
+ RSPEC_MATCHING_JS_FILES_PATH
+ ].freeze
+
+ def initialize
+ missing_env_variables = REQUIRED_ENV_VARIABLES.select { |key| ENV[key.to_s] == '' }
+ unless missing_env_variables.empty?
+ raise "[predictive tests] Missing ENV variable(s): #{missing_env_variables.join(',')}."
+ end
+
+ @rspec_changed_files_path = ENV['RSPEC_CHANGED_FILES_PATH']
+ @rspec_matching_tests_path = ENV['RSPEC_MATCHING_TESTS_PATH']
+ @rspec_views_including_partials_path = ENV['RSPEC_VIEWS_INCLUDING_PARTIALS_PATH']
+ @frontend_fixtures_mapping_path = ENV['FRONTEND_FIXTURES_MAPPING_PATH']
+ @rspec_matching_js_files_path = ENV['RSPEC_MATCHING_JS_FILES_PATH']
+ end
+
+ def execute
+ Tooling::FindChanges.new(
+ from: :api,
+ changed_files_pathname: rspec_changed_files_path
+ ).execute
+ Tooling::FindFilesUsingFeatureFlags.new(changed_files_pathname: rspec_changed_files_path).execute
+ Tooling::FindTests.new(rspec_changed_files_path, rspec_matching_tests_path).execute
+ Tooling::Mappings::PartialToViewsMappings.new(
+ rspec_changed_files_path, rspec_views_including_partials_path).execute
+ Tooling::FindTests.new(rspec_views_including_partials_path, rspec_matching_tests_path).execute
+ Tooling::Mappings::JsToSystemSpecsMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute
+ Tooling::Mappings::GraphqlBaseTypeMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute
+ Tooling::Mappings::ViewToSystemSpecsMappings.new(rspec_changed_files_path, rspec_matching_tests_path).execute
+ Tooling::FindChanges.new(
+ from: :changed_files,
+ changed_files_pathname: rspec_changed_files_path,
+ predictive_tests_pathname: rspec_matching_tests_path,
+ frontend_fixtures_mapping_pathname: frontend_fixtures_mapping_path
+ ).execute
+ Tooling::Mappings::ViewToJsMappings.new(rspec_changed_files_path, rspec_matching_js_files_path).execute
+ end
+
+ private
+
+ attr_reader :rspec_changed_files_path, :rspec_matching_tests_path, :rspec_views_including_partials_path,
+ :frontend_fixtures_mapping_path, :rspec_matching_js_files_path
+ end
+end
diff --git a/tooling/lib/tooling/test_map_generator.rb b/tooling/lib/tooling/test_map_generator.rb
index f96f33ff074..88b4353b232 100644
--- a/tooling/lib/tooling/test_map_generator.rb
+++ b/tooling/lib/tooling/test_map_generator.rb
@@ -12,7 +12,9 @@ module Tooling
def parse(yaml_files)
Array(yaml_files).each do |yaml_file|
data = File.read(yaml_file)
- metadata, example_groups = data.split("---\n").reject(&:empty?).map { |yml| YAML.safe_load(yml, [Symbol]) }
+ metadata, example_groups = data.split("---\n").reject(&:empty?).map do |yml|
+ YAML.safe_load(yml, permitted_classes: [Symbol])
+ end
if example_groups.nil?
puts "No examples in #{yaml_file}! Metadata: #{metadata}"
diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb
index eeda135f3ee..20e00763f65 100644
--- a/tooling/quality/test_level.rb
+++ b/tooling/quality/test_level.rb
@@ -18,6 +18,7 @@ module Quality
unit: %w[
bin
channels
+ components
config
contracts
db
@@ -54,7 +55,6 @@ module Quality
views
workers
tooling
- components
],
integration: %w[
commands
@@ -77,8 +77,8 @@ module Quality
@patterns[level] ||= "#{prefixes_for_pattern}spec/#{folders_pattern(level)}{,/**/}*#{suffix(level)}".freeze # rubocop:disable Style/RedundantFreeze
end
- def regexp(level)
- @regexps[level] ||= Regexp.new("#{prefixes_for_regex}spec/#{folders_regex(level)}").freeze
+ def regexp(level, start_with = false)
+ @regexps[level] ||= Regexp.new("#{'^' if start_with}#{prefixes_for_regex}spec/#{folders_regex(level)}").freeze
end
def level_for(file_path)
diff --git a/tooling/rspec_flaky/config.rb b/tooling/rspec_flaky/config.rb
index 36e35671587..0e36e985aad 100644
--- a/tooling/rspec_flaky/config.rb
+++ b/tooling/rspec_flaky/config.rb
@@ -18,10 +18,6 @@ module RspecFlaky
ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec/flaky/new-report.json")
end
- def self.skipped_flaky_tests_report_path
- ENV['SKIPPED_FLAKY_TESTS_REPORT_PATH'] || rails_path("rspec/flaky/skipped_flaky_tests_report.txt")
- end
-
def self.rails_path(path)
return path unless defined?(Rails)