From 7e9c479f7de77702622631cff2628a9c8dcbc627 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Thu, 19 Nov 2020 08:27:35 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-6-stable-ee --- tooling/README.md | 4 ++ tooling/eslint-config/conditionally_ignore_ee.js | 5 ++ .../coverage_lines_execution_detector.rb | 44 +++++++++++++++ .../tooling/crystalball/coverage_lines_strategy.rb | 23 ++++++++ tooling/lib/tooling/helm3_client.rb | 3 +- tooling/lib/tooling/test_map_generator.rb | 36 +++++++++++++ tooling/lib/tooling/test_map_packer.rb | 58 ++++++++++++++++++++ tooling/overcommit/Gemfile | 2 +- tooling/overcommit/Gemfile.lock | 63 ++++++++++++---------- 9 files changed, 206 insertions(+), 32 deletions(-) create mode 100644 tooling/README.md create mode 100644 tooling/eslint-config/conditionally_ignore_ee.js create mode 100644 tooling/lib/tooling/crystalball/coverage_lines_execution_detector.rb create mode 100644 tooling/lib/tooling/crystalball/coverage_lines_strategy.rb create mode 100644 tooling/lib/tooling/test_map_generator.rb create mode 100644 tooling/lib/tooling/test_map_packer.rb (limited to 'tooling') diff --git a/tooling/README.md b/tooling/README.md new file mode 100644 index 00000000000..29706a5af14 --- /dev/null +++ b/tooling/README.md @@ -0,0 +1,4 @@ +# Tooling + +This directory contains tools and configuration for development only. + diff --git a/tooling/eslint-config/conditionally_ignore_ee.js b/tooling/eslint-config/conditionally_ignore_ee.js new file mode 100644 index 00000000000..e5e3c8013f4 --- /dev/null +++ b/tooling/eslint-config/conditionally_ignore_ee.js @@ -0,0 +1,5 @@ +/* eslint-disable import/no-commonjs */ + +const IS_EE = require('../../config/helpers/is_ee_env'); + +module.exports = IS_EE ? {} : { ignorePatterns: ['ee/**/*.*'] }; diff --git a/tooling/lib/tooling/crystalball/coverage_lines_execution_detector.rb b/tooling/lib/tooling/crystalball/coverage_lines_execution_detector.rb new file mode 100644 index 00000000000..47ddf568fe4 --- /dev/null +++ b/tooling/lib/tooling/crystalball/coverage_lines_execution_detector.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'crystalball/map_generator/helpers/path_filter' + +module Tooling + module Crystalball + # Class for detecting code execution path based on coverage information diff + class CoverageLinesExecutionDetector + include ::Crystalball::MapGenerator::Helpers::PathFilter + + attr_reader :exclude_prefixes + + def initialize(*args, exclude_prefixes: []) + super(*args) + @exclude_prefixes = exclude_prefixes + end + + # Detects files affected during example execution based on line coverage. + # Transforms absolute paths to relative. + # Exclude paths outside of repository and in excluded prefixes + # + # @param[Hash] hash of files affected before example execution + # @param[Hash] hash of files affected after example execution + # @return [Array] + def detect(before, after) + file_names = after.keys + covered_files = file_names.reject { |file_name| same_coverage?(before, after, file_name) } + filter(covered_files) + end + + private + + def same_coverage?(before, after, file_name) + before[file_name] && before[file_name][:lines] == after[file_name][:lines] + end + + def filter(paths) + super.reject do |file_name| + exclude_prefixes.any? { |prefix| file_name.start_with?(prefix) } + end + end + end + end +end diff --git a/tooling/lib/tooling/crystalball/coverage_lines_strategy.rb b/tooling/lib/tooling/crystalball/coverage_lines_strategy.rb new file mode 100644 index 00000000000..ebcaab0b8d8 --- /dev/null +++ b/tooling/lib/tooling/crystalball/coverage_lines_strategy.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'coverage' +require 'crystalball/map_generator/coverage_strategy' +require_relative './coverage_lines_execution_detector' + +module Tooling + module Crystalball + # Crystalball map generator strategy based on Crystalball::MapGenerator::CoverageStrategy, + # modified to use Coverage.start(lines: true) + # This maintains compatibility with SimpleCov on Ruby >= 2.5 with start arguments + # and SimpleCov.start uses Coverage.start(lines: true) by default + class CoverageLinesStrategy < ::Crystalball::MapGenerator::CoverageStrategy + def initialize(execution_detector = CoverageLinesExecutionDetector) + super(execution_detector) + end + + def after_register + Coverage.start(lines: true) + end + end + end +end diff --git a/tooling/lib/tooling/helm3_client.rb b/tooling/lib/tooling/helm3_client.rb index 802ff9b9661..d6671688794 100644 --- a/tooling/lib/tooling/helm3_client.rb +++ b/tooling/lib/tooling/helm3_client.rb @@ -3,7 +3,6 @@ require 'time' require 'json' require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen) -require_relative '../../../lib/gitlab/json' unless defined?(Gitlab::Json) module Tooling class Helm3Client @@ -67,7 +66,7 @@ module Tooling %(--output json), *args ] - releases = Gitlab::Json.parse(run_command(command)) + releases = JSON.parse(run_command(command)) # rubocop:disable Gitlab/Json releases.map do |release| Release.new(*release.values_at(*RELEASE_JSON_ATTRIBUTES)) diff --git a/tooling/lib/tooling/test_map_generator.rb b/tooling/lib/tooling/test_map_generator.rb new file mode 100644 index 00000000000..bd0415f6e67 --- /dev/null +++ b/tooling/lib/tooling/test_map_generator.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'set' +require 'yaml' + +module Tooling + class TestMapGenerator + def initialize + @mapping = Hash.new { |h, k| h[k] = Set.new } + end + + def parse(yaml_files) + Array(yaml_files).each do |yaml_file| + data = File.read(yaml_file) + _metadata, example_groups = data.split("---\n").reject(&:empty?).map { |yml| YAML.safe_load(yml, [Symbol]) } + + example_groups.each do |example_id, files| + files.each do |file| + spec_file = strip_example_uid(example_id) + @mapping[file] << spec_file + end + end + end + end + + def mapping + @mapping.transform_values { |set| set.to_a } + end + + private + + def strip_example_uid(example_id) + example_id.gsub(/\[.+\]/, '') + end + end +end diff --git a/tooling/lib/tooling/test_map_packer.rb b/tooling/lib/tooling/test_map_packer.rb new file mode 100644 index 00000000000..520d69610eb --- /dev/null +++ b/tooling/lib/tooling/test_map_packer.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Tooling + class TestMapPacker + SEPARATOR = '/'.freeze + MARKER = 1 + + def pack(map) + map.transform_values(&method(:create_tree_from_tests)) + end + + def unpack(compact_map) + compact_map.transform_values(&method(:retrieve_tests_from_tree)) + end + + private + + def create_tree_from_tests(tests) + tests.inject({}) do |tree, test| + segments = test.split(SEPARATOR) + branch = create_branch_from_segments(segments) + deep_merge(tree, branch) + end + end + + def create_branch_from_segments(segments) + segments.reverse.inject(MARKER) { |node, parent| { parent => node } } + end + + def deep_merge(hash, other) + hash.merge(other) do |_, this_val, other_val| + if this_val.is_a?(Hash) && other_val.is_a?(Hash) + deep_merge(this_val, other_val) + else + other_val + end + end + end + + def retrieve_tests_from_tree(tree) + traverse(tree).inject([]) do |tests, test| + tests << test + end + end + + def traverse(tree, segments = [], &block) + return to_enum(__method__, tree, segments) unless block_given? + + if tree == MARKER + return yield segments.join(SEPARATOR) + end + + tree.each do |key, value| + traverse(value, segments + [key], &block) + end + end + end +end diff --git a/tooling/overcommit/Gemfile b/tooling/overcommit/Gemfile index 615da316fd5..08f08018ffb 100644 --- a/tooling/overcommit/Gemfile +++ b/tooling/overcommit/Gemfile @@ -4,6 +4,6 @@ source 'https://rubygems.org' gem 'overcommit' -gem 'gitlab-styles', '~> 4.3.0', require: false +gem 'gitlab-styles', '~> 5.1.0', require: false gem 'scss_lint', '~> 0.56.0', require: false gem 'haml_lint', '~> 0.34.0', require: false diff --git a/tooling/overcommit/Gemfile.lock b/tooling/overcommit/Gemfile.lock index d3a98855d96..5bada88e1dc 100644 --- a/tooling/overcommit/Gemfile.lock +++ b/tooling/overcommit/Gemfile.lock @@ -1,22 +1,22 @@ GEM remote: https://rubygems.org/ specs: - activesupport (6.0.3) + activesupport (6.0.3.4) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) zeitwerk (~> 2.2, >= 2.2.2) - ast (2.4.0) + ast (2.4.1) childprocess (3.0.0) - concurrent-ruby (1.1.6) + concurrent-ruby (1.1.7) ffi (1.12.2) - gitlab-styles (4.3.0) - rubocop (~> 0.82.0) + gitlab-styles (5.1.0) + rubocop (~> 0.89.1) rubocop-gitlab-security (~> 0.1.0) - rubocop-performance (~> 1.5.2) - rubocop-rails (~> 2.5) - rubocop-rspec (~> 1.36) + rubocop-performance (~> 1.8.1) + rubocop-rails (~> 2.8) + rubocop-rspec (~> 1.44) haml (5.1.2) temple (>= 0.8.0) tilt @@ -25,42 +25,47 @@ GEM rainbow rubocop (>= 0.50.0) sysexits (~> 1.1) - i18n (1.8.2) + i18n (1.8.5) concurrent-ruby (~> 1.0) iniparse (1.5.0) - jaro_winkler (1.5.4) - minitest (5.11.3) + minitest (5.14.2) overcommit (0.53.0) childprocess (>= 0.6.3, < 4) iniparse (~> 1.4) - parallel (1.19.1) - parser (2.7.1.2) - ast (~> 2.4.0) - rack (2.0.9) + parallel (1.19.2) + parser (2.7.2.0) + ast (~> 2.4.1) + rack (2.2.3) rainbow (3.0.0) rake (12.3.3) rb-fsevent (0.10.2) rb-inotify (0.9.10) ffi (>= 0.5.0, < 2) + regexp_parser (1.8.2) rexml (3.2.4) - rubocop (0.82.0) - jaro_winkler (~> 1.5.1) + rubocop (0.89.1) parallel (~> 1.10) - parser (>= 2.7.0.1) + parser (>= 2.7.1.1) rainbow (>= 2.2.2, < 4.0) + regexp_parser (>= 1.7) rexml + rubocop-ast (>= 0.3.0, < 1.0) ruby-progressbar (~> 1.7) unicode-display_width (>= 1.4.0, < 2.0) + rubocop-ast (0.8.0) + parser (>= 2.7.1.5) rubocop-gitlab-security (0.1.1) rubocop (>= 0.51) - rubocop-performance (1.5.2) - rubocop (>= 0.71.0) - rubocop-rails (2.5.2) - activesupport + rubocop-performance (1.8.1) + rubocop (>= 0.87.0) + rubocop-ast (>= 0.4.0) + rubocop-rails (2.8.1) + activesupport (>= 4.2.0) rack (>= 1.1) - rubocop (>= 0.72.0) - rubocop-rspec (1.37.0) - rubocop (>= 0.68.1) + rubocop (>= 0.87.0) + rubocop-rspec (1.44.1) + rubocop (~> 0.87) + rubocop-ast (>= 0.7.1) ruby-progressbar (1.10.1) sass (3.5.5) sass-listen (~> 4.0.0) @@ -74,19 +79,19 @@ GEM temple (0.8.2) thread_safe (0.3.6) tilt (2.0.10) - tzinfo (1.2.7) + tzinfo (1.2.8) thread_safe (~> 0.1) unicode-display_width (1.7.0) - zeitwerk (2.3.0) + zeitwerk (2.4.1) PLATFORMS ruby DEPENDENCIES - gitlab-styles (~> 4.3.0) + gitlab-styles (~> 5.1.0) haml_lint (~> 0.34.0) overcommit scss_lint (~> 0.56.0) BUNDLED WITH - 1.17.3 + 2.1.4 -- cgit v1.2.3