diff options
Diffstat (limited to 'qa/qa/tools/reliable_report.rb')
| -rw-r--r-- | qa/qa/tools/reliable_report.rb | 299 |
1 files changed, 210 insertions, 89 deletions
diff --git a/qa/qa/tools/reliable_report.rb b/qa/qa/tools/reliable_report.rb index 9d2079171c1..40a452be36e 100644 --- a/qa/qa/tools/reliable_report.rb +++ b/qa/qa/tools/reliable_report.rb @@ -1,111 +1,223 @@ # frozen_string_literal: true +require_relative "../../qa" + require "influxdb-client" require "terminal-table" require "slack-notifier" +require "colorize" module QA module Tools class ReliableReport - def initialize(run_type, range = 30) - @results = 10 - @slack_channel = "#quality-reports" + include Support::API + + # Project for report creation: https://gitlab.com/gitlab-org/gitlab + PROJECT_ID = 278964 + + def initialize(range) @range = range - @run_type = run_type - @stable_title = "Top #{results} stable specs for past #{@range} days in '#{run_type}' runs" - @unstable_title = "Top #{results} unstable reliable specs for past #{@range} days in '#{run_type}' runs" + @influxdb_bucket = "e2e-test-stats" + @slack_channel = "#quality-reports" + @influxdb_url = ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL env variable") + @influxdb_token = ENV["QA_INFLUXDB_TOKEN"] || raise("Missing QA_INFLUXDB_TOKEN env variable") end - # Print top stable specs + # Run reliable reporter # + # @param [Integer] range amount of days for results range + # @param [String] report_in_issue_and_slack # @return [void] - def show_top_stable - puts terminal_table( - rows: top_stable.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] }, - title: stable_title - ) + def self.run(range: 14, report_in_issue_and_slack: "false") + reporter = new(range) + + reporter.print_report + reporter.report_in_issue_and_slack if report_in_issue_and_slack == "true" + rescue StandardError => e + puts "Report creation failed! Error: '#{e}'".colorize(:red) + reporter.notify_failure(e) + exit(1) end - # Post top stable spec report to slack - # Slice table in to multiple messages due to max char limitation + # Print top stable specs # # @return [void] - def notify_top_stable - tables = top_stable.each_slice(5).map do |slice| - terminal_table( - rows: slice.map { |spec| [name_column(spec[0], spec[1][:file]), *table_params(spec[1].values)] } - ) - end + def print_report + puts "#{stable_summary_table}\n\n" + stable_results_tables.each { |stage, table| puts "#{table}\n\n" } + return puts("No unstable reliable tests present!".colorize(:yellow)) if unstable_reliable_test_runs.empty? - puts "\nSending top stable spec report to #{slack_channel} slack channel" - slack_args = { icon_emoji: ":mtg_green:", username: "Stable Spec Report" } - notifier.post(text: "*#{stable_title}*", **slack_args) - tables.each { |table| notifier.post(text: "```#{table}```", **slack_args) } + puts "#{unstable_summary_table}\n\n" + unstable_reliable_results_tables.each { |stage, table| puts "#{table}\n\n" } end - # Print top unstable specs + # Create report issue # # @return [void] - def show_top_unstable - return puts("No unstable tests present!") if top_unstable_reliable.empty? + def report_in_issue_and_slack + puts "Creating report".colorize(:green) + response = post( + "#{gitlab_api_url}/projects/#{PROJECT_ID}/issues", + { title: "Reliable spec report", description: report_issue_body, labels: "Quality,test" }, + headers: { "PRIVATE-TOKEN" => gitlab_access_token } + ) + web_url = parse_body(response)[:web_url] + puts "Created report issue: #{web_url}" - puts terminal_table( - rows: top_unstable_reliable.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] }, - title: unstable_title + puts "Sending slack notification".colorize(:green) + notifier.post( + icon_emoji: ":tanuki-protect:", + text: <<~TEXT + ```#{stable_summary_table}``` + ```#{unstable_summary_table}``` + + #{web_url} + TEXT ) + puts "Done!" end - # Post top unstable reliable spec report to slack - # Slice table in to multiple messages due to max char limitation + # Notify failure # + # @param [StandardError] error # @return [void] - def notify_top_unstable - return puts("No unstable tests present!") if top_unstable_reliable.empty? + def notify_failure(error) + notifier.post( + text: "Reliable reporter failed to create report. Error: ```#{error}```", + icon_emoji: ":sadpanda:" + ) + end - tables = top_unstable_reliable.each_slice(5).map do |slice| - terminal_table( - rows: slice.map { |spec| [name_column(spec[0], spec[1][:file]), *table_params(spec[1].values)] } - ) - end + private - puts "\nSending top unstable reliable spec report to #{slack_channel} slack channel" - slack_args = { icon_emoji: ":sadpanda:", username: "Unstable Spec Report" } - notifier.post(text: "*#{unstable_title}*", **slack_args) - tables.each { |table| notifier.post(text: "```#{table}```", **slack_args) } + attr_reader :range, :influxdb_bucket, :slack_channel, :influxdb_url, :influxdb_token + + # Markdown formatted report issue body + # + # @return [String] + def report_issue_body + issue = [] + issue << "[[_TOC_]]" + issue << "# Candidates for promotion to reliable\n\n```\n#{stable_summary_table}\n```" + issue << results_markdown(stable_results_tables) + return issue.join("\n\n") if unstable_reliable_test_runs.empty? + + issue << "# Reliable specs with failures\n\n```\n#{unstable_summary_table}\n```" + issue << results_markdown(unstable_reliable_results_tables) + issue.join("\n\n") end - private + # Stable spec summary table + # + # @return [Terminal::Table] + def stable_summary_table + @stable_summary_table ||= terminal_table( + rows: stable_test_runs.map { |stage, specs| [stage, specs.length] }, + title: "Stable spec summary for past #{range} days".ljust(50), + headings: %w[STAGE COUNT] + ) + end - attr_reader :results, - :slack_channel, - :range, - :run_type, - :stable_title, - :unstable_title + # Unstable reliable summary table + # + # @return [Terminal::Table] + def unstable_summary_table + @unstable_summary_table ||= terminal_table( + rows: unstable_reliable_test_runs.map { |stage, specs| [stage, specs.length] }, + title: "Unstable spec summary for past #{range} days".ljust(50), + headings: %w[STAGE COUNT] + ) + end - # Top stable specs + # Result tables for stable specs # # @return [Hash] - def top_stable - @top_stable ||= runs(reliable: false).sort_by { |k, v| [v[:failure_rate], -v[:runs]] }[0..results - 1].to_h + def stable_results_tables + @stable_results ||= results_tables(:stable) end - # Top unstable reliable specs + # Result table for unstable specs # # @return [Hash] - def top_unstable_reliable - @top_unstable_reliable ||= runs(reliable: true) - .reject { |k, v| v[:failure_rate] == 0 } - .sort_by { |k, v| -v[:failure_rate] }[0..results - 1] - .to_h + def unstable_reliable_results_tables + @unstable_results ||= results_tables(:unstable) + end + + # Markdown formatted tables + # + # @param [Hash] results + # @return [String] + def results_markdown(results) + results.map do |stage, table| + <<~STAGE.strip + ## #{stage} + + <details> + <summary>Executions table</summary> + + ``` + #{table} + ``` + + </details> + STAGE + end.join("\n\n") + end + + # Results table + # + # @param [Symbol] type result type - :stable, :unstable + # @return [Hash<Symbol, Terminal::Table>] + def results_tables(type) + (type == :stable ? stable_test_runs : unstable_reliable_test_runs).to_h do |stage, specs| + headings = ["name", "runs", "failures", "failure rate"] + + [stage, terminal_table( + rows: specs.map { |k, v| [name_column(k, v[:file]), *table_params(v.values)] }, + title: "Top #{type} specs in '#{stage}' stage for past #{range} days", + headings: headings.map(&:upcase) + )] + end + end + + # Stable specs + # + # @return [Hash] + def stable_test_runs + @top_stable ||= begin + stable_specs = test_runs(reliable: false).transform_values do |specs| + specs + .reject { |k, v| v[:failure_rate] != 0 } + .sort_by { |k, v| -v[:runs] } + .to_h + end + + stable_specs.reject { |k, v| v.empty? } + end + end + + # Unstable reliable specs + # + # @return [Hash] + def unstable_reliable_test_runs + @top_unstable_reliable ||= begin + unstable = test_runs(reliable: true).transform_values do |specs| + specs + .reject { |k, v| v[:failure_rate] == 0 } + .sort_by { |k, v| -v[:failure_rate] } + .to_h + end + + unstable.reject { |k, v| v.empty? } + end end # Terminal table for result formatting # # @return [Terminal::Table] - def terminal_table(rows:, title: nil) + def terminal_table(rows:, headings:, title: nil) Terminal::Table.new( - headings: ["name", "runs", "failed", "failure rate"], + headings: headings, style: { all_separators: true }, title: title, rows: rows @@ -126,30 +238,32 @@ module QA # @param [String] file # @return [String] def name_column(name, file) - spec_name = name.length > 100 ? "#{name} ".scan(/.{1,100} /).map(&:strip).join("\n") : name + spec_name = name.length > 150 ? "#{name} ".scan(/.{1,150} /).map(&:strip).join("\n") : name name_line = "name: '#{spec_name}'" file_line = "file: '#{file}'" - "#{name_line}\n#{file_line.ljust(110)}" + "#{name_line}\n#{file_line.ljust(160)}" end # Test executions grouped by name # # @param [Boolean] reliable - # @return [Hash] - def runs(reliable:) - puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past 30 days in '#{run_type}' runs") - puts + # @return [Hash<String, Hash>] + def test_runs(reliable:) + puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past #{range} days\n".colorize(:green)) - query_api.query(query: query(reliable)).values.each_with_object({}) do |table, result| + all_runs = query_api.query(query: query(reliable)).values + all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result| records = table.records name = records.last.values["name"] file = records.last.values["file_path"].split("/").last + stage = records.last.values["stage"] || "unknown" + runs = records.count failed = records.count { |r| r.values["status"] == "failed" } failure_rate = (failed.to_f / runs.to_f) * 100 - result[name] = { + result[stage][name] = { file: file, runs: runs, failed: failed, @@ -164,17 +278,24 @@ module QA # @return [String] def query(reliable) <<~QUERY - from(bucket: "e2e-test-stats") - |> range(start: -#{range}d) - |> filter(fn: (r) => r._measurement == "test-stats" and - r.run_type == "#{run_type}" and - r.status != "pending" and - r.merge_request == "false" and - r.quarantined == "false" and - r.reliable == "#{reliable}" and - r._field == "id" - ) - |> group(columns: ["name"]) + from(bucket: "#{influxdb_bucket}") + |> range(start: -#{range}d) + |> filter(fn: (r) => r._measurement == "test-stats") + |> filter(fn: (r) => r.run_type == "staging-full" or + r.run_type == "staging-sanity" or + r.run_type == "staging-sanity-no-admin" or + r.run_type == "production-full" or + r.run_type == "production-sanity" or + r.run_type == "package-and-qa" or + r.run_type == "nightly" + ) + |> filter(fn: (r) => r.status != "pending" and + r.merge_request == "false" and + r.quarantined == "false" and + r.reliable == "#{reliable}" and + r._field == "id" + ) + |> group(columns: ["name"]) QUERY end @@ -192,7 +313,7 @@ module QA @influx_client ||= InfluxDB2::Client.new( influxdb_url, influxdb_token, - bucket: "e2e-test-stats", + bucket: influxdb_bucket, org: "gitlab-qa", precision: InfluxDB2::WritePrecision::NANOSECOND ) @@ -205,29 +326,29 @@ module QA @notifier ||= Slack::Notifier.new( slack_webhook_url, channel: slack_channel, - username: "Reliable spec reporter" + username: "Reliable Spec Report" ) end - # InfluxDb instance url + # Gitlab access token # # @return [String] - def influxdb_url - @influxdb_url ||= ENV["QA_INFLUXDB_URL"] || raise("Missing QA_INFLUXDB_URL environment variable") + def gitlab_access_token + @gitlab_access_token ||= ENV["GITLAB_ACCESS_TOKEN"] || raise("Missing GITLAB_ACCESS_TOKEN env variable") end - # Influxdb token + # Gitlab api url # # @return [String] - def influxdb_token - @influxdb_token ||= ENV["QA_INFLUXDB_TOKEN"] || raise("Missing QA_INFLUXDB_TOKEN environment variable") + def gitlab_api_url + @gitlab_api_url ||= ENV["CI_API_V4_URL"] || raise("Missing CI_API_V4_URL env variable") end # Slack webhook url # # @return [String] def slack_webhook_url - @slack_webhook_url ||= ENV["CI_SLACK_WEBHOOK_URL"] || raise("Missing CI_SLACK_WEBHOOK_URL environment variable") + @slack_webhook_url ||= ENV["SLACK_WEBHOOK"] || raise("Missing SLACK_WEBHOOK env variable") end end end |