From b39512ed755239198a9c294b6a45e65c05900235 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 18 Aug 2022 08:17:02 +0000 Subject: Add latest changes from gitlab-org/gitlab@15-3-stable-ee --- tooling/bin/qa/package_and_qa_check | 45 ----------------- tooling/bin/qa/run_qa_check | 45 +++++++++++++++++ tooling/config/CODEOWNERS.yml | 26 +++++++++- tooling/danger/customer_success.rb | 45 +++++++++++++++++ tooling/danger/product_intelligence.rb | 2 +- tooling/danger/project_helper.rb | 2 + tooling/graphql/docs/helper.rb | 5 +- tooling/lib/tooling/find_codeowners.rb | 92 ++++++++++++++++++++++------------ tooling/lib/tooling/helm3_client.rb | 4 +- tooling/lib/tooling/test_map_packer.rb | 2 +- tooling/overcommit/Gemfile | 8 --- tooling/overcommit/Gemfile.lock | 83 ------------------------------ tooling/overcommit/Makefile | 7 --- tooling/quality/test_level.rb | 1 - 14 files changed, 184 insertions(+), 183 deletions(-) delete mode 100755 tooling/bin/qa/package_and_qa_check create mode 100755 tooling/bin/qa/run_qa_check create mode 100644 tooling/danger/customer_success.rb delete mode 100644 tooling/overcommit/Gemfile delete mode 100644 tooling/overcommit/Gemfile.lock delete mode 100644 tooling/overcommit/Makefile (limited to 'tooling') diff --git a/tooling/bin/qa/package_and_qa_check b/tooling/bin/qa/package_and_qa_check deleted file mode 100755 index 21deb0fcd2d..00000000000 --- a/tooling/bin/qa/package_and_qa_check +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'pathname' - -# This script checks if the package-and-qa job should trigger downstream pipelines to run the QA suite. -# -# It assumes the first argument is a directory of files containing diffs of changes from an MR -# (e.g., created by tooling/bin/find_change_diffs). It exits with a success code if there are no diffs, or if the diffs -# are suitable to run QA tests. -# -# The script will abort (exit code 1) if the argument is missing. -# -# The following condition will result in a failure code (2), indicating that package-and-qa should not run: -# -# - If the changes only include tests being put in quarantine - -abort("ERROR: Please specify the directory containing MR diffs.") if ARGV.empty? -diffs_dir = Pathname.new(ARGV.shift).expand_path - -# Run package-and-qa if there are no diffs. E.g., in scheduled pipelines -exit 0 if diffs_dir.glob('**/*').empty? - -files_count = 0 -specs_count = 0 -quarantine_specs_count = 0 - -diffs_dir.glob('**/*').each do |path| - next if path.directory? - - files_count += 1 - next unless path.to_s.end_with?('_spec.rb.diff') - - specs_count += 1 - quarantine_specs_count += 1 if path.read.match?(/^\+.*, quarantine:/) -end - -# Run package-and-qa if there are no specs. E.g., when the MR changes QA framework files. -exit 0 if specs_count == 0 - -# Skip package-and-qa if there are only specs being put in quarantine. -exit 2 if quarantine_specs_count == specs_count && quarantine_specs_count == files_count - -# Run package-and-qa under any other circumstances. E.g., if there are specs being put in quarantine but there are also -# other changes that might need to be tested. diff --git a/tooling/bin/qa/run_qa_check b/tooling/bin/qa/run_qa_check new file mode 100755 index 00000000000..5b8844ec4fd --- /dev/null +++ b/tooling/bin/qa/run_qa_check @@ -0,0 +1,45 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'pathname' + +# This script checks if the code changes justify running the QA suite. +# +# It assumes the first argument is a directory of files containing diffs of changes from an MR +# (e.g., created by tooling/bin/find_change_diffs). It exits with a success code if there are no diffs, or if the diffs +# are suitable to run QA tests. +# +# The script will abort (exit code 1) if the argument is missing. +# +# The following condition will result in a failure code (2), indicating that QA tests should not run: +# +# - If the changes only include tests being put in quarantine + +abort("ERROR: Please specify the directory containing MR diffs.") if ARGV.empty? +diffs_dir = Pathname.new(ARGV.shift).expand_path + +# Run QA tests if there are no diffs. E.g., in scheduled pipelines +exit 0 if diffs_dir.glob('**/*').empty? + +files_count = 0 +specs_count = 0 +quarantine_specs_count = 0 + +diffs_dir.glob('**/*').each do |path| + next if path.directory? + + files_count += 1 + next unless path.to_s.end_with?('_spec.rb.diff') + + specs_count += 1 + quarantine_specs_count += 1 if path.read.match?(/^\+.*,? quarantine:/) +end + +# Run QA tests if there are no specs. E.g., when the MR changes QA framework files. +exit 0 if specs_count == 0 + +# Skip QA tests if there are only specs being put in quarantine. +exit 2 if quarantine_specs_count == specs_count && quarantine_specs_count == files_count + +# Run QA tests under any other circumstances. E.g., if there are specs being put in quarantine but there are also +# other changes that might need to be tested. diff --git a/tooling/config/CODEOWNERS.yml b/tooling/config/CODEOWNERS.yml index a8ae90437e2..71818b67ab1 100644 --- a/tooling/config/CODEOWNERS.yml +++ b/tooling/config/CODEOWNERS.yml @@ -3,7 +3,7 @@ # And paste the contents into .gitlab/CODEOWNERS '[Authentication and Authorization]': - '@gitlab-org/manage/authentication-and-authorization': + '@gitlab-org/manage/authentication-and-authorization/approvers': allow: keywords: - 'password' @@ -55,3 +55,27 @@ - '/lib/gitlab/conan_token.rb' patterns: - '%{keyword}' + +'[Compliance]': + '@gitlab-org/manage/compliance': + entries: + - '/ee/app/services/audit_events/build_service.rb' + - '/ee/spec/services/audit_events/custom_audit_event_service_spec.rb' + allow: + keywords: + - audit + patterns: + - '**%{keyword}**' + deny: + keywords: + - '*.png' + - '*bundler-audit*' + - '**/merge_requests/**' + - '/ee/app/services/audit_events/*' + - '/ee/config/feature_flags/development/auditor_group_runner_access.yml' + - '/ee/spec/services/audit_events/*' + - '/ee/spec/services/ci/*' + - '/ee/spec/services/personal_access_tokens/*' + - '/qa/**/*' + patterns: + - '%{keyword}' diff --git a/tooling/danger/customer_success.rb b/tooling/danger/customer_success.rb new file mode 100644 index 00000000000..43bdeadd8a4 --- /dev/null +++ b/tooling/danger/customer_success.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Tooling + module Danger + module CustomerSuccess + CHANGED_SCHEMA_MESSAGE = <<~MSG + Notification to the Customer Success about changes to files with possible breaking downstream processes, add label `Customer Success::Impact Check`. + + /label ~"Customer Success::Impact Check" + + The following files require a review: + MSG + + FILE_PATH_REGEX = %r{((ee|jh)/)?config/metrics/.+\.yml}.freeze + CATEGORY_CHANGED = /data_category: operational/i.freeze + + def build_message + return unless impacted? + + CHANGED_SCHEMA_MESSAGE + helper.markdown_list(impacted_files) + end + + private + + def impacted? + !helper.has_scoped_label_with_scope?('Customer Success') && impacted_files.any? + end + + def impacted_files + @impacted_files ||= + metric_files.select do |file| + helper.changed_lines(file).any? { |change| metric_category_changed?(change) } + end.compact + end + + def metric_files + helper.modified_files.grep(FILE_PATH_REGEX) + end + + def metric_category_changed?(change) + change =~ CATEGORY_CHANGED + end + end + end +end diff --git a/tooling/danger/product_intelligence.rb b/tooling/danger/product_intelligence.rb index 0f007e970b4..621a7b509b0 100644 --- a/tooling/danger/product_intelligence.rb +++ b/tooling/danger/product_intelligence.rb @@ -7,7 +7,7 @@ module Tooling APPROVED_LABEL = 'product intelligence::approved' REVIEW_LABEL = 'product intelligence::review pending' CHANGED_FILES_MESSAGE = <<~MSG - For the following files, a review from the [Data team and Product Intelligence team](https://gitlab.com/groups/gitlab-org/growth/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) is recommended + For the following files, a review from the [Data team and Product Intelligence team](https://gitlab.com/groups/gitlab-org/analytics-section/product-intelligence/engineers/-/group_members?with_inherited_permissions=exclude) is recommended Please check the ~"product intelligence" [Service Ping guide](https://docs.gitlab.com/ee/development/service_ping/) or the [Snowplow guide](https://docs.gitlab.com/ee/development/snowplow/). For MR review guidelines, see the [Service Ping review guidelines](https://docs.gitlab.com/ee/development/service_ping/review_guidelines.html) or the [Snowplow review guidelines](https://docs.gitlab.com/ee/development/snowplow/review_guidelines.html). diff --git a/tooling/danger/project_helper.rb b/tooling/danger/project_helper.rb index b37b86ceecc..d8c7d617927 100644 --- a/tooling/danger/project_helper.rb +++ b/tooling/danger/project_helper.rb @@ -63,6 +63,7 @@ module Tooling %r{\A((ee|jh)/)?app/views/} => [:frontend, :backend], %r{\A((ee|jh)/)?public/} => :frontend, %r{\A((ee|jh)/)?spec/(javascripts|frontend|frontend_integration)/} => :frontend, + %r{\A((ee|jh)/)?spec/contracts/consumer} => :frontend, %r{\A((ee|jh)/)?vendor/assets/} => :frontend, %r{\A((ee|jh)/)?scripts/frontend/} => :frontend, %r{(\A|/)( @@ -117,6 +118,7 @@ module Tooling %r{\Alib/gitlab/ci/templates} => :ci_template, %r{\A((ee|jh)/)?spec/features/} => :test, + %r{\A((ee|jh)/)?spec/contracts/} => :test, %r{\A((ee|jh)/)?spec/support/shared_examples/features/} => :test, %r{\A((ee|jh)/)?spec/support/shared_contexts/features/} => :test, %r{\A((ee|jh)/)?spec/support/helpers/features/} => :test, diff --git a/tooling/graphql/docs/helper.rb b/tooling/graphql/docs/helper.rb index e4f14129f3b..a76773ed28d 100644 --- a/tooling/graphql/docs/helper.rb +++ b/tooling/graphql/docs/helper.rb @@ -315,14 +315,17 @@ module Tooling def render_deprecation(object, owner, context) buff = [] deprecation = schema_deprecation(owner, object[:name]) + original_description = deprecation&.original_description || render_description_of(object, owner) - buff << (deprecation&.original_description || render_description_of(object, owner)) if context == :block + buff << original_description if context == :block buff << if deprecation deprecation.markdown(context: context) else "**Deprecated:** #{object[:deprecation_reason]}" end + buff << original_description if context == :inline && deprecation&.alpha? + join(context, buff) end diff --git a/tooling/lib/tooling/find_codeowners.rb b/tooling/lib/tooling/find_codeowners.rb index 3b50b33d85c..6a90f86eecc 100644 --- a/tooling/lib/tooling/find_codeowners.rb +++ b/tooling/lib/tooling/find_codeowners.rb @@ -9,37 +9,10 @@ module Tooling puts section group_defintions.each do |group, list| - matched_files = git_ls_files.each_line.select do |line| - list[:allow].find do |pattern| - path = "/#{line.chomp}" + print_entries(group, list[:entries]) if list[:entries] + print_expanded_entries(group, list) if list[:allow] - path_matches?(pattern, path) && - list[:deny].none? { |pattern| path_matches?(pattern, path) } - end - end - - consolidated = consolidate_paths(matched_files) - consolidated_again = consolidate_paths(consolidated) - - # Consider the directory structure is a tree structure: - # https://en.wikipedia.org/wiki/Tree_(data_structure) - # After we consolidated the leaf entries, it could be possible that - # we can consolidate further for the new leaves. Repeat this - # process until we see no improvements. - while consolidated_again.size < consolidated.size - consolidated = consolidated_again - consolidated_again = consolidate_paths(consolidated) - end - - consolidated.each do |line| - path = line.chomp - - if File.directory?(path) - puts "/#{path}/ #{group}" - else - puts "/#{path} #{group}" - end - end + puts end end end @@ -50,10 +23,20 @@ module Tooling result.each do |section, group_defintions| group_defintions.each do |group, definitions| definitions.transform_values! do |rules| - rules[:keywords].flat_map do |keyword| - rules[:patterns].map do |pattern| - pattern % { keyword: keyword } + case rules + when Hash + case rules[:keywords] + when Array + rules[:keywords].flat_map do |keyword| + rules[:patterns].map do |pattern| + pattern % { keyword: keyword } + end + end + else + rules[:patterns] end + when Array + rules end end end @@ -118,6 +101,49 @@ module Tooling private + def print_entries(group, entries) + entries.each do |entry| + puts "#{entry} #{group}" + end + end + + def print_expanded_entries(group, list) + matched_files = git_ls_files.each_line.select do |line| + list[:allow].find do |pattern| + path = "/#{line.chomp}" + + path_matches?(pattern, path) && + ( + list[:deny].nil? || + list[:deny].none? { |pattern| path_matches?(pattern, path) } + ) + end + end + + consolidated = consolidate_paths(matched_files) + consolidated_again = consolidate_paths(consolidated) + + # Consider the directory structure is a tree structure: + # https://en.wikipedia.org/wiki/Tree_(data_structure) + # After we consolidated the leaf entries, it could be possible that + # we can consolidate further for the new leaves. Repeat this + # process until we see no improvements. + while consolidated_again.size < consolidated.size + consolidated = consolidated_again + consolidated_again = consolidate_paths(consolidated) + end + + consolidated.each do |line| + path = line.chomp + + if File.directory?(path) + puts "/#{path}/ #{group}" + else + puts "/#{path} #{group}" + end + end + end + def find_dir_maxdepth_1(dir) `find #{dir} -maxdepth 1` end diff --git a/tooling/lib/tooling/helm3_client.rb b/tooling/lib/tooling/helm3_client.rb index 6e4a35e82f1..82ebe3f51dc 100644 --- a/tooling/lib/tooling/helm3_client.rb +++ b/tooling/lib/tooling/helm3_client.rb @@ -84,7 +84,7 @@ module Tooling # method - The Octokit method to use for getting the data. # args - Arguments to pass to the `helm list` command. def each_releases_page(args, &block) - return to_enum(__method__, args) unless block_given? + return to_enum(__method__, args) unless block page = 0 final_args = args.dup @@ -100,7 +100,7 @@ module Tooling # # args - Any arguments to pass to the `helm list` command. def each_release(args, &block) - return to_enum(__method__, args) unless block_given? + return to_enum(__method__, args) unless block each_releases_page(args) do |page| page.releases.each do |release| diff --git a/tooling/lib/tooling/test_map_packer.rb b/tooling/lib/tooling/test_map_packer.rb index d74edb9500f..151ce88111f 100644 --- a/tooling/lib/tooling/test_map_packer.rb +++ b/tooling/lib/tooling/test_map_packer.rb @@ -44,7 +44,7 @@ module Tooling end def traverse(tree, segments = [], &block) - return to_enum(__method__, tree, segments) unless block_given? + return to_enum(__method__, tree, segments) unless block if tree == MARKER return yield segments.join(SEPARATOR) diff --git a/tooling/overcommit/Gemfile b/tooling/overcommit/Gemfile deleted file mode 100644 index 26dad738bab..00000000000 --- a/tooling/overcommit/Gemfile +++ /dev/null @@ -1,8 +0,0 @@ -# frozen_string_literal: true - -# Make sure to run `make` (in this directory) when you update this file. -source 'https://rubygems.org' - -gem 'overcommit' -gem 'gitlab-styles', '~> 5.4.0', require: false -gem 'haml_lint', '~> 0.34.0', require: false diff --git a/tooling/overcommit/Gemfile.lock b/tooling/overcommit/Gemfile.lock deleted file mode 100644 index a4b6f62a979..00000000000 --- a/tooling/overcommit/Gemfile.lock +++ /dev/null @@ -1,83 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - activesupport (6.0.3.4) - concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) - zeitwerk (~> 2.2, >= 2.2.2) - ast (2.4.1) - childprocess (3.0.0) - concurrent-ruby (1.1.7) - gitlab-styles (5.4.0) - rubocop (~> 0.89.1) - rubocop-gitlab-security (~> 0.1.0) - rubocop-performance (~> 1.8.1) - rubocop-rails (~> 2.8) - rubocop-rspec (~> 1.44) - haml (5.1.2) - temple (>= 0.8.0) - tilt - haml_lint (0.34.0) - haml (>= 4.0, < 5.2) - rainbow - rubocop (>= 0.50.0) - sysexits (~> 1.1) - i18n (1.8.5) - concurrent-ruby (~> 1.0) - iniparse (1.5.0) - minitest (5.14.2) - overcommit (0.53.0) - childprocess (>= 0.6.3, < 4) - iniparse (~> 1.4) - parallel (1.19.2) - parser (2.7.2.0) - ast (~> 2.4.1) - rack (2.2.3.1) - rainbow (3.0.0) - regexp_parser (1.8.2) - rexml (3.2.4) - rubocop (0.89.1) - parallel (~> 1.10) - parser (>= 2.7.1.1) - rainbow (>= 2.2.2, < 4.0) - regexp_parser (>= 1.7) - rexml - rubocop-ast (>= 0.3.0, < 1.0) - ruby-progressbar (~> 1.7) - unicode-display_width (>= 1.4.0, < 2.0) - rubocop-ast (0.8.0) - parser (>= 2.7.1.5) - rubocop-gitlab-security (0.1.1) - rubocop (>= 0.51) - rubocop-performance (1.8.1) - rubocop (>= 0.87.0) - rubocop-ast (>= 0.4.0) - rubocop-rails (2.8.1) - activesupport (>= 4.2.0) - rack (>= 1.1) - rubocop (>= 0.87.0) - rubocop-rspec (1.44.1) - rubocop (~> 0.87) - rubocop-ast (>= 0.7.1) - ruby-progressbar (1.10.1) - sysexits (1.2.0) - temple (0.8.2) - thread_safe (0.3.6) - tilt (2.0.10) - tzinfo (1.2.8) - thread_safe (~> 0.1) - unicode-display_width (1.7.0) - zeitwerk (2.4.1) - -PLATFORMS - ruby - -DEPENDENCIES - gitlab-styles (~> 5.4.0) - haml_lint (~> 0.34.0) - overcommit - -BUNDLED WITH - 2.1.4 diff --git a/tooling/overcommit/Makefile b/tooling/overcommit/Makefile deleted file mode 100644 index 7da3c7c0569..00000000000 --- a/tooling/overcommit/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -.PHONY: install - -install: - if [ -e ../../.overcommit.yml ]; then cp -f ../../.overcommit.yml ../../.overcommit.yml.backup; fi - cp ../../.overcommit.yml.example ../../.overcommit.yml - bundle install - cd ../../ && overcommit -i diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb index 82da0121e31..d630ffd5432 100644 --- a/tooling/quality/test_level.rb +++ b/tooling/quality/test_level.rb @@ -24,7 +24,6 @@ module Quality elastic elastic_integration experiments - events factories finders frontend -- cgit v1.2.3