#!/usr/bin/env ruby # frozen_string_literal: true # We don't have auto-loading here require_relative '../lib/gitlab' require_relative '../lib/gitlab/popen' require_relative '../lib/gitlab/popen/runner' class StaticAnalysis # `ALLOWED_WARNINGS` moved to scripts/allowed_warnings.txt Task = Struct.new(:command, :duration) do def cmd command.join(' ') end end NodeAssignment = Struct.new(:index, :tasks) do def total_duration return 0 if tasks.empty? tasks.sum(&:duration) end end def self.project_path project_root = File.expand_path('..', __dir__) if Gitlab.jh? "#{project_root}/jh" else project_root end end # `gettext:updated_check` and `gitlab:sidekiq:sidekiq_queues_yml:check` will fail on FOSS installations # (e.g. gitlab-org/gitlab-foss) since they test against a single # file that is generated by an EE installation, which can # contain values that a FOSS installation won't find. To work # around this we will only enable this task on EE installations. TASKS_WITH_DURATIONS_SECONDS = [ (Gitlab.ee? ? Task.new(%w[bin/rake gettext:updated_check], 360) : nil), Task.new(%w[yarn run lint:prettier], 200), Task.new(%w[bin/rake gettext:lint], 105), Task.new(%W[scripts/license-check.sh #{project_path}], 200), Task.new(%w[bin/rake lint:static_verification], 40), Task.new(%w[bin/rake config_lint], 10), Task.new(%w[bin/rake gitlab:sidekiq:all_queues_yml:check], 15), (Gitlab.ee? ? Task.new(%w[bin/rake gitlab:sidekiq:sidekiq_queues_yml:check], 11) : nil), Task.new(%w[yarn run internal:stylelint], 8), Task.new(%w[scripts/lint-conflicts.sh], 1), Task.new(%w[yarn run block-dependencies], 1), Task.new(%w[yarn run check-dependencies], 1), Task.new(%w[scripts/gemfile_lock_changed.sh], 1) ].compact.freeze def run_tasks!(options = {}) total_nodes = (ENV['CI_NODE_TOTAL'] || 1).to_i current_node_number = (ENV['CI_NODE_INDEX'] || 1).to_i node_assignment = tasks_to_run(total_nodes)[current_node_number - 1] if options[:dry_run] puts "Dry-run mode!" return end static_analysis = Gitlab::Popen::Runner.new start_time = Time.now static_analysis.run(node_assignment.tasks.map(&:command)) do |command, &run| task = node_assignment.tasks.find { |task| task.command == command } puts puts "$ #{task.cmd}" result = run.call puts "==> Finished in #{result.duration} seconds (expected #{task.duration} seconds)" puts end puts puts '===================================================' puts "Node finished running all tasks in #{Time.now - start_time} seconds (expected #{node_assignment.total_duration})" puts puts if static_analysis.all_success_and_clean? puts 'All static analyses passed successfully.' elsif static_analysis.all_success? puts 'All static analyses passed successfully with warnings.' puts emit_warnings(static_analysis) # We used to exit 2 on warnings but `fail_on_warnings` takes care of it now. else puts 'Some static analyses failed:' emit_warnings(static_analysis) emit_errors(static_analysis) exit 1 end end def emit_warnings(static_analysis) static_analysis.warned_results.each do |result| warn warn "**** #{result.cmd.join(' ')} had the following warning(s):" warn warn result.stderr warn end end def emit_errors(static_analysis) static_analysis.failed_results.each do |result| puts puts "**** #{result.cmd.join(' ')} failed with the following error(s):" puts puts result.stdout puts result.stderr puts end end def tasks_to_run(node_total) total_time = TASKS_WITH_DURATIONS_SECONDS.sum(&:duration).to_f ideal_time_per_node = total_time / node_total tasks_by_duration_desc = TASKS_WITH_DURATIONS_SECONDS.sort_by { |a| -a.duration } nodes = Array.new(node_total) { |i| NodeAssignment.new(i + 1, []) } puts "Total expected time: #{total_time}; ideal time per job: #{ideal_time_per_node}.\n\n" puts "Tasks to distribute:" tasks_by_duration_desc.each { |task| puts "* #{task.cmd} (#{task.duration}s)" } # Distribute tasks optimally first puts "\nAssigning tasks optimally." distribute_tasks(tasks_by_duration_desc, nodes, ideal_time_per_node: ideal_time_per_node) # Distribute remaining tasks, ordered by ascending duration leftover_tasks = tasks_by_duration_desc - nodes.flat_map(&:tasks) if leftover_tasks.any? puts "\n\nAssigning remaining tasks: #{leftover_tasks.flat_map(&:cmd)}" distribute_tasks(leftover_tasks, nodes.sort_by { |node| node.total_duration }) end nodes.each do |node| puts "\nExpected duration for node #{node.index}: #{node.total_duration} seconds" node.tasks.each { |task| puts "* #{task.cmd} (#{task.duration}s)" } end nodes end def distribute_tasks(tasks, nodes, ideal_time_per_node: nil) condition = if ideal_time_per_node ->(task, node, ideal_time_per_node) { (task.duration + node.total_duration) <= ideal_time_per_node } else ->(*) { true } end tasks.each do |task| nodes.each do |node| if condition.call(task, node, ideal_time_per_node) assign_task_to_node(tasks, node, task) break end end end end def assign_task_to_node(remaining_tasks, node, task) node.tasks << task puts "Assigning #{task.command} (#{task.duration}s) to node ##{node.index}. Node total duration: #{node.total_duration}s." end end if $PROGRAM_NAME == __FILE__ options = {} if ARGV.include?('--dry-run') options[:dry_run] = true end StaticAnalysis.new.run_tasks!(options) end