diff options
Diffstat (limited to 'scripts')
-rw-r--r-- | scripts/api/commit_merge_requests.rb | 29 | ||||
-rwxr-xr-x | scripts/create-pipeline-failure-incident.rb | 90 | ||||
-rwxr-xr-x | scripts/generate-failed-pipeline-slack-message.rb | 2 |
3 files changed, 118 insertions, 3 deletions
diff --git a/scripts/api/commit_merge_requests.rb b/scripts/api/commit_merge_requests.rb new file mode 100644 index 00000000000..3cf8dc87497 --- /dev/null +++ b/scripts/api/commit_merge_requests.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'gitlab' +require_relative 'default_options' + +class CommitMergeRequests + def initialize(options) + @project = options.fetch(:project) + @sha = options.fetch(:sha) + + # If api_token is nil, it's set to '' to allow unauthenticated requests (for forks). + api_token = options.fetch(:api_token, '') + + warn "No API token given." if api_token.empty? + + @client = Gitlab.client( + endpoint: options.fetch(:endpoint, API::DEFAULT_OPTIONS[:endpoint]), + private_token: api_token + ) + end + + def execute + client.commit_merge_requests(project, sha) + end + + private + + attr_reader :project, :sha, :client +end diff --git a/scripts/create-pipeline-failure-incident.rb b/scripts/create-pipeline-failure-incident.rb index 1035a680291..2b86fac680d 100755 --- a/scripts/create-pipeline-failure-incident.rb +++ b/scripts/create-pipeline-failure-incident.rb @@ -5,9 +5,10 @@ require 'optparse' require 'json' -require_relative 'api/pipeline_failed_jobs' +require_relative 'api/commit_merge_requests' require_relative 'api/create_issue' require_relative 'api/create_issue_discussion' +require_relative 'api/pipeline_failed_jobs' class CreatePipelineFailureIncident DEFAULT_OPTIONS = { @@ -29,6 +30,8 @@ class CreatePipelineFailureIncident labels: incident_labels } + payload[:assignee_ids] = assignee_ids if stable_branch_incident? + CreateIssue.new(project: project, api_token: api_token).execute(payload).tap do |incident| CreateIssueDiscussion.new(project: project, api_token: api_token) .execute(issue_iid: incident.iid, body: "## Root Cause Analysis") @@ -41,8 +44,18 @@ class CreatePipelineFailureIncident attr_reader :project, :api_token + def stable_branch_incident? + ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/ + end + def failed_jobs - @failed_jobs ||= PipelineFailedJobs.new(API::DEFAULT_OPTIONS.dup.merge(exclude_allowed_to_fail_jobs: true)).execute + @failed_jobs ||= PipelineFailedJobs.new(API::DEFAULT_OPTIONS.merge(exclude_allowed_to_fail_jobs: true)).execute + end + + def merge_request + @merge_request ||= CommitMergeRequests.new( + API::DEFAULT_OPTIONS.merge(sha: ENV['CI_COMMIT_SHA']) + ).execute.first end def now @@ -63,6 +76,12 @@ class CreatePipelineFailureIncident end def description + return broken_stable_description_content if stable_branch_incident? + + broken_master_description_content + end + + def broken_master_description_content <<~MARKDOWN ## #{project_link} pipeline #{pipeline_link} failed @@ -107,7 +126,62 @@ class CreatePipelineFailureIncident MARKDOWN end + def broken_stable_description_content + <<~MARKDOWN + ## #{project_link} pipeline #{pipeline_link} failed + + **Branch: #{branch_link}** + + **Commit: #{commit_link}** + + **Merge Request: #{merge_request_link}** + + **Triggered by** #{triggered_by_link} • **Source:** #{source} • **Duration:** #{pipeline_duration} minutes + + **Failed jobs (#{failed_jobs.size}):** + + #{failed_jobs_list} + + ### General guidelines + + A broken stable branch prevents patch releases from being built. + Fixing the pipeline is a priority to prevent any delays in releases. + + The process in the [Broken `master` handbook guide](https://about.gitlab.com/handbook/engineering/workflow/#broken-master) can be referenced since much of that process also applies here. + + ### Investigation + + **Be sure to fill the `Timeline` for this incident.** + + 1. If the failure is new, and looks like a potential flaky failure, you can retry the failing job. + Make sure to mention the retry in the `Timeline` and leave a link to the retried job. + 1. Search for similar master-broken issues in https://gitlab.com/gitlab-org/quality/engineering-productivity/master-broken-incidents/-/issues + 1. If one exists, ask the DRI of the master-broken issue to cherry-pick any resulting merge requests into the stable branch + + @gitlab-org/release/managers if the merge request author or maintainer is not available, this can be escalated using the dev-on-call process in the [#dev-escalation slack channel](https://gitlab.slack.com/archives/CLKLMSUR4). + + ### Pre-resolution + + If you believe that there's an easy resolution by either: + + - Reverting a particular merge request. + - Making a quick fix (for example, one line or a few similar simple changes in a few lines). + You can create a merge request, assign to any available maintainer, and ping people that were involved/related to the introduction of the failure. + Additionally, a message can be posted in `#backend_maintainers` or `#frontend_maintainers` to get a maintainer take a look at the fix ASAP. + - Cherry picking a change that was used to fix a similar master-broken issue. + + In both cases, make sure to add the ~"pipeline:expedite" label to speed up the `stable`-fixing pipelines. + + ### Resolution + + Add a comment to this issue describing how this incident could have been prevented earlier in the Merge Request pipeline (rather than the merge commit pipeline). + + MARKDOWN + end + def incident_labels + return ['release-blocker'] if stable_branch_incident? + master_broken_label = if ENV['CI_PROJECT_NAME'] == 'gitlab-foss' 'master:foss-broken' @@ -118,6 +192,12 @@ class CreatePipelineFailureIncident DEFAULT_LABELS.dup << master_broken_label end + def assignee_ids + ids = [ENV['GITLAB_USER_ID'].to_i] + ids << merge_request['author']['id'].to_i if merge_request + ids + end + def pipeline_link "[##{ENV['CI_PIPELINE_ID']}](#{ENV['CI_PIPELINE_URL']})" end @@ -134,6 +214,12 @@ class CreatePipelineFailureIncident "[#{ENV['CI_COMMIT_TITLE']}](#{ENV['CI_PROJECT_URL']}/-/commit/#{ENV['CI_COMMIT_SHA']})" end + def merge_request_link + return 'N/A' unless merge_request + + "[#{merge_request['title']}](#{merge_request['web_url']})" + end + def source "`#{ENV['CI_PIPELINE_SOURCE']}`" end diff --git a/scripts/generate-failed-pipeline-slack-message.rb b/scripts/generate-failed-pipeline-slack-message.rb index b695cdfdbee..eefdebd5db5 100755 --- a/scripts/generate-failed-pipeline-slack-message.rb +++ b/scripts/generate-failed-pipeline-slack-message.rb @@ -107,7 +107,7 @@ class GenerateFailedPipelineSlackMessage if incident_exist? incident['web_url'] else - "#{ENV['CI_SERVER_URL']}/#{ENV['BROKEN_MASTER_INCIDENTS_PROJECT']}/-/issues/new?" \ + "#{ENV['CI_SERVER_URL']}/#{ENV['BROKEN_BRANCH_INCIDENTS_PROJECT']}/-/issues/new?" \ "issuable_template=incident&issue%5Bissue_type%5D=incident" end end |