diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-03-21 15:08:46 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-03-21 15:08:46 +0300 |
commit | 7f521d27811b472c43203ed3d1bde4460a617f89 (patch) | |
tree | 47f1a10b776991e86c6db002bc6e03e83acc356a /tooling | |
parent | 83e3316a189d3b709b23af30647b5f9ea5377bac (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'tooling')
-rwxr-xr-x | tooling/bin/partial_to_views_mappings | 9 | ||||
-rw-r--r-- | tooling/lib/tooling/kubernetes_client.rb | 156 | ||||
-rw-r--r-- | tooling/lib/tooling/mappings/partial_to_views_mappings.rb | 105 |
3 files changed, 156 insertions, 114 deletions
diff --git a/tooling/bin/partial_to_views_mappings b/tooling/bin/partial_to_views_mappings new file mode 100755 index 00000000000..12c994ee556 --- /dev/null +++ b/tooling/bin/partial_to_views_mappings @@ -0,0 +1,9 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require_relative '../lib/tooling/mappings/partial_to_views_mappings' + +changes_file = ARGV.shift +output_file = ARGV.shift + +Tooling::Mappings::PartialToViewsMappings.new(changes_file, output_file).execute diff --git a/tooling/lib/tooling/kubernetes_client.rb b/tooling/lib/tooling/kubernetes_client.rb index 27eb4c8151e..b373f5d6980 100644 --- a/tooling/lib/tooling/kubernetes_client.rb +++ b/tooling/lib/tooling/kubernetes_client.rb @@ -6,78 +6,49 @@ require_relative '../../../lib/gitlab/popen' unless defined?(Gitlab::Popen) module Tooling class KubernetesClient - RESOURCE_LIST = 'ingress,svc,pdb,hpa,deploy,statefulset,job,pod,secret,configmap,pvc,secret,clusterrole,clusterrolebinding,role,rolebinding,sa,crd' K8S_ALLOWED_NAMESPACES_REGEX = /^review-(?!apps).+/.freeze CommandFailedError = Class.new(StandardError) - attr_reader :namespace + def cleanup_pvcs_by_created_at(created_before:) + stale_pvcs = pvcs_created_before(created_before: created_before) - def initialize(namespace:) - @namespace = namespace - end + # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby. + review_apps_stale_pvcs = stale_pvcs.select do |stale_pvc_hash| + K8S_ALLOWED_NAMESPACES_REGEX.match?(stale_pvc_hash[:namespace]) + end + return if review_apps_stale_pvcs.empty? - def cleanup_by_release(release_name:, wait: true) - delete_by_selector(release_name: release_name, wait: wait) - delete_by_matching_name(release_name: release_name) + review_apps_stale_pvcs.each do |pvc_hash| + delete_pvc(pvc_hash[:resource_name], pvc_hash[:namespace]) + end end - def cleanup_by_created_at(resource_type:, created_before:, wait: true) - resource_names = resource_names_created_before(resource_type: resource_type, created_before: created_before) - return if resource_names.empty? + def cleanup_namespaces_by_created_at(created_before:) + stale_namespaces = namespaces_created_before(created_before: created_before) - delete_by_exact_names(resource_type: resource_type, resource_names: resource_names, wait: wait) - end - - def cleanup_review_app_namespaces(created_before:, wait: true) - namespaces = review_app_namespaces_created_before(created_before: created_before) - return if namespaces.empty? + # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby. + review_apps_stale_namespaces = stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) } + return if review_apps_stale_namespaces.empty? - delete_namespaces_by_exact_names(resource_names: namespaces, wait: wait) + delete_namespaces(review_apps_stale_namespaces) end - def delete_namespaces_by_exact_names(resource_names:, wait:) - command = [ - 'delete', - 'namespace', - '--now', - '--ignore-not-found', - %(--wait=#{wait}), - resource_names.join(' ') - ] + def delete_pvc(pvc, namespace) + return unless K8S_ALLOWED_NAMESPACES_REGEX.match?(namespace) - run_command(command) + run_command("kubectl delete pvc --namespace=#{namespace} --now --ignore-not-found #{pvc}") end - private - - def delete_by_selector(release_name:, wait:) - selector = case release_name - when String - %(-l release="#{release_name}") - when Array - %(-l 'release in (#{release_name.join(', ')})') - else - raise ArgumentError, 'release_name must be a string or an array' - end - - command = [ - 'delete', - RESOURCE_LIST, - %(--namespace "#{namespace}"), - '--now', - '--ignore-not-found', - %(--wait=#{wait}), - selector - ] + def delete_namespaces(namespaces) + return if namespaces.any? { |ns| !K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) } - run_command(command) + run_command("kubectl delete namespace --now --ignore-not-found #{namespaces.join(' ')}") end - def delete_by_exact_names(resource_names:, wait:, resource_type: nil) + def delete_namespaces_by_exact_names(resource_names:, wait:) command = [ 'delete', - resource_type, - %(--namespace "#{namespace}"), + 'namespace', '--now', '--ignore-not-found', %(--wait=#{wait}), @@ -87,87 +58,44 @@ module Tooling run_command(command) end - def delete_by_matching_name(release_name:) - resource_names = raw_resource_names - command = [ - 'delete', - %(--namespace "#{namespace}"), - '--ignore-not-found' - ] - - Array(release_name).each do |release| - resource_names - .select { |resource_name| resource_name.include?(release) } - .each { |matching_resource| run_command(command + [matching_resource]) } + def pvcs_created_before(created_before:) + resource_created_before(resource_type: 'pvc', created_before: created_before) do |item| + { + resource_name: item.dig('metadata', 'name'), + namespace: item.dig('metadata', 'namespace') + } end end - def raw_resource_names - command = [ - 'get', - RESOURCE_LIST, - %(--namespace "#{namespace}"), - '-o name' - ] - run_command(command).lines.map(&:strip) - end - - def resource_names_created_before(resource_type:, created_before:) - command = [ - 'get', - resource_type, - %(--namespace "#{namespace}"), - "--sort-by='{.metadata.creationTimestamp}'", - '-o json' - ] - - response = run_command(command) - - resources_created_before_date(response, created_before) + def namespaces_created_before(created_before:) + resource_created_before(resource_type: 'namespace', created_before: created_before) do |item| + item.dig('metadata', 'name') + end end - def review_app_namespaces_created_before(created_before:) - command = [ - 'get', - 'namespace', - "--sort-by='{.metadata.creationTimestamp}'", - '-o json' - ] - - response = run_command(command) - - stale_namespaces = resources_created_before_date(response, created_before) - - # `kubectl` doesn't allow us to filter namespaces with a regexp. We therefore do the filtering in Ruby. - stale_namespaces.select { |ns| K8S_ALLOWED_NAMESPACES_REGEX.match?(ns) } - end + def resource_created_before(resource_type:, created_before:) + response = run_command("kubectl get #{resource_type} --all-namespaces --sort-by='{.metadata.creationTimestamp}' -o json") - def resources_created_before_date(response, date) items = JSON.parse(response)['items'] # rubocop:disable Gitlab/Json - - items.each_with_object([]) do |item, result| + items.filter_map do |item| item_created_at = Time.parse(item.dig('metadata', 'creationTimestamp')) - if item_created_at < date - resource_name = item.dig('metadata', 'name') - result << resource_name - end + yield item if item_created_at < created_before end rescue ::JSON::ParserError => ex - puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" # rubocop:disable Rails/Output + puts "Ignoring this JSON parsing error: #{ex}\n\nResponse was:\n#{response}" [] end def run_command(command) - final_command = ['kubectl', *command.compact].join(' ') - puts "Running command: `#{final_command}`" # rubocop:disable Rails/Output + puts "Running command: `#{command}`" - result = Gitlab::Popen.popen_with_detail([final_command]) + result = Gitlab::Popen.popen_with_detail([command]) if result.status.success? result.stdout.chomp.freeze else - raise CommandFailedError, "The `#{final_command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}" + raise CommandFailedError, "The `#{command}` command failed (status: #{result.status}) with the following error:\n#{result.stderr}" end end end diff --git a/tooling/lib/tooling/mappings/partial_to_views_mappings.rb b/tooling/lib/tooling/mappings/partial_to_views_mappings.rb new file mode 100644 index 00000000000..1b36894d881 --- /dev/null +++ b/tooling/lib/tooling/mappings/partial_to_views_mappings.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +require_relative 'base' +require_relative '../../../../lib/gitlab_edition' + +# Returns view files that include the potential rails partials from the changed files passed as input. +module Tooling + module Mappings + class PartialToViewsMappings < Base + def initialize(changes_file, output_file, view_base_folder: 'app/views') + @output_file = output_file + @changed_files = File.read(changes_file).split(' ') + @view_base_folders = folders_for_available_editions(view_base_folder) + end + + def execute + views_including_modified_partials = [] + + views_globs = view_base_folders.map { |view_base_folder| "#{view_base_folder}/**/*.html.haml" } + Dir[*views_globs].each do |view_file| + included_partial_names = find_pattern_in_file(view_file, partials_keywords_regexp) + next if included_partial_names.empty? + + included_partial_names.each do |included_partial_name| + if view_includes_modified_partial?(view_file, included_partial_name) + views_including_modified_partials << view_file + end + end + end + + File.write(output_file, views_including_modified_partials.join(' ')) + end + + def filter_files + @_filter_files ||= changed_files.select do |filename| + filename.start_with?(*view_base_folders) && + File.basename(filename).start_with?('_') && + File.basename(filename).end_with?('.html.haml') && + File.exist?(filename) + end + end + + def partials_keywords_regexp + partial_keywords = filter_files.map do |partial_filename| + extract_partial_keyword(partial_filename) + end + + partial_regexps = partial_keywords.map do |keyword| + %r{(?:render|render_if_exists)(?: |\()(?:partial: ?)?['"]([\w\-_/]*#{keyword})['"]} + end + + Regexp.union(partial_regexps) + end + + # e.g. if app/views/clusters/clusters/_sidebar.html.haml was modified, the partial keyword is `sidebar`. + def extract_partial_keyword(partial_filename) + File.basename(partial_filename).delete_prefix('_').delete_suffix('.html.haml') + end + + # Why do we need this method? + # + # Assume app/views/clusters/clusters/_sidebar.html.haml was modified in the MR. + # + # Suppose now you find = render 'sidebar' in a view. Is this view including the sidebar partial + # that was modified, or another partial called "_sidebar.html.haml" somewhere else? + def view_includes_modified_partial?(view_file, included_partial_name) + view_file_parent_folder = File.dirname(view_file) + included_partial_filename = reconstruct_partial_filename(included_partial_name) + included_partial_relative_path = File.join(view_file_parent_folder, included_partial_filename) + + # We do this because in render (or render_if_exists) + # apparently looks for partials in other GitLab editions + # + # Example: + # + # ee/app/views/events/_epics_filter.html.haml is used in app/views/shared/_event_filter.html.haml + # with render_if_exists 'events/epics_filter' + included_partial_absolute_paths = view_base_folders.map do |view_base_folder| + File.join(view_base_folder, included_partial_filename) + end + + filter_files.include?(included_partial_relative_path) || + (filter_files & included_partial_absolute_paths).any? + end + + def reconstruct_partial_filename(partial_name) + partial_path = partial_name.split('/')[..-2] + partial_filename = partial_name.split('/').last + full_partial_filename = "_#{partial_filename}.html.haml" + + return full_partial_filename if partial_path.empty? + + File.join(partial_path.join('/'), full_partial_filename) + end + + def find_pattern_in_file(file, pattern) + File.read(file).scan(pattern).flatten.uniq + end + + private + + attr_reader :changed_files, :output_file, :view_base_folders + end + end +end |