diff options
Diffstat (limited to 'scripts/trigger-build.rb')
| -rwxr-xr-x | scripts/trigger-build.rb | 484 |
1 files changed, 484 insertions, 0 deletions
diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb new file mode 100755 index 00000000000..17cbd91a8ee --- /dev/null +++ b/scripts/trigger-build.rb @@ -0,0 +1,484 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'gitlab' + +module Trigger + def self.ee? + # Support former project name for `dev` + %w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME']) + end + + def self.security? + %r{\Agitlab-org/security(\z|/)}.match?(ENV['CI_PROJECT_NAMESPACE']) + end + + def self.non_empty_variable_value(variable) + variable_value = ENV[variable] + + return if variable_value.nil? || variable_value.empty? + + variable_value + end + + def self.variables_for_env_file(variables) + variables.map do |key, value| + %Q(#{key}=#{value}) + end.join("\n") + end + + class Base + # Can be overridden + def self.access_token + ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN'] + end + + def invoke!(post_comment: false, downstream_job_name: nil) + pipeline_variables = variables + + puts "Triggering downstream pipeline on #{downstream_project_path}" + puts "with variables #{pipeline_variables}" + + pipeline = gitlab_client(:downstream).run_trigger( + downstream_project_path, + trigger_token, + ref, + pipeline_variables) + + puts "Triggered downstream pipeline: #{pipeline.web_url}\n" + puts "Waiting for downstream pipeline status" + + Trigger::CommitComment.post!(pipeline, gitlab_client(:upstream)) if post_comment + downstream_job = + if downstream_job_name + gitlab_client(:downstream).pipeline_jobs(downstream_project_path, pipeline.id).auto_paginate.find do |potential_job| + potential_job.name == downstream_job_name + end + end + + if downstream_job + Trigger::Job.new(downstream_project_path, downstream_job.id, gitlab_client(:downstream)) + else + Trigger::Pipeline.new(downstream_project_path, pipeline.id, gitlab_client(:downstream)) + end + end + + def variables + simple_forwarded_variables.merge(base_variables, extra_variables, version_file_variables) + end + + def simple_forwarded_variables + { + 'TRIGGER_SOURCE' => ENV['CI_JOB_URL'], + 'TOP_UPSTREAM_SOURCE_PROJECT' => ENV['CI_PROJECT_PATH'], + 'TOP_UPSTREAM_SOURCE_REF' => ENV['CI_COMMIT_REF_NAME'], + 'TOP_UPSTREAM_SOURCE_JOB' => ENV['CI_JOB_URL'], + 'TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID' => ENV['CI_MERGE_REQUEST_PROJECT_ID'], + 'TOP_UPSTREAM_MERGE_REQUEST_IID' => ENV['CI_MERGE_REQUEST_IID'] + } + end + + private + + # Override to trigger and work with pipeline on different GitLab instance + # type: :downstream -> downstream build and pipeline status + # type: :upstream -> this project, e.g. for posting comments + def gitlab_client(type) + # By default, always use the same client + @gitlab_client ||= Gitlab.client( + endpoint: 'https://gitlab.com/api/v4', + private_token: self.class.access_token + ) + end + + # Must be overridden + def downstream_project_path + raise NotImplementedError + end + + # Must be overridden + def ref + raise NotImplementedError + end + + # Can be overridden + def trigger_token + ENV['CI_JOB_TOKEN'] + end + + # Can be overridden + def extra_variables + {} + end + + # Can be overridden + def version_param_value(version_file) + ENV[version_file]&.strip || File.read(version_file).strip + end + + def base_variables + # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results, + # and fallback to CI_COMMIT_SHA for the `detached` pipelines. + { + 'GITLAB_REF_SLUG' => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_REF_SLUG'], + 'TRIGGERED_USER' => ENV['TRIGGERED_USER'] || ENV['GITLAB_USER_NAME'], + 'TOP_UPSTREAM_SOURCE_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'] + } + end + + # Read version files from all components + def version_file_variables + Dir.glob("*_VERSION").each_with_object({}) do |version_file, params| + params[version_file] = version_param_value(version_file) + end + end + end + + class Omnibus < Base + def self.access_token + # Default to "Multi-pipeline (from 'gitlab-org/gitlab' 'package-and-qa' job)" at https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/settings/access_tokens + ENV['OMNIBUS_GITLAB_PROJECT_ACCESS_TOKEN'] || super + end + + private + + def downstream_project_path + ENV.fetch('OMNIBUS_PROJECT_PATH', 'gitlab-org/build/omnibus-gitlab-mirror') + end + + def ref + ENV.fetch('OMNIBUS_BRANCH', 'master') + end + + def extra_variables + # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA (MR HEAD commit) so that the image is in sync with the assets and QA images. + # See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results. + # We also set IMAGE_TAG so the GitLab Docker image is tagged with that SHA. + source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'] + + { + 'GITLAB_VERSION' => source_sha, + 'IMAGE_TAG' => source_sha, + 'QA_IMAGE' => ENV['QA_IMAGE'], + 'SKIP_QA_DOCKER' => 'true', + 'ALTERNATIVE_SOURCES' => 'true', + 'SECURITY_SOURCES' => Trigger.security? ? 'true' : 'false', + 'ee' => Trigger.ee? ? 'true' : 'false', + 'QA_BRANCH' => ENV['QA_BRANCH'] || 'master', + 'CACHE_UPDATE' => ENV['OMNIBUS_GITLAB_CACHE_UPDATE'], + 'GITLAB_QA_OPTIONS' => ENV['GITLAB_QA_OPTIONS'], + 'QA_TESTS' => ENV['QA_TESTS'], + 'ALLURE_JOB_NAME' => ENV['ALLURE_JOB_NAME'] + } + end + end + + class CNG < Base + def variables + # Delete variables that aren't useful when using native triggers. + super.tap do |hash| + hash.delete('TRIGGER_SOURCE') + hash.delete('TRIGGERED_USER') + end + end + + private + + def ref + return ENV['CI_COMMIT_REF_NAME'] if ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/ + + ENV.fetch('CNG_BRANCH', 'master') + end + + def extra_variables + # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA (MR HEAD commit) so that the image is in sync with the assets and QA images. + source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'] + + { + "TRIGGER_BRANCH" => ref, + "GITLAB_VERSION" => source_sha, + "GITLAB_TAG" => ENV['CI_COMMIT_TAG'], # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. + "GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : source_sha, + "FORCE_RAILS_IMAGE_BUILDS" => 'true', + "CE_PIPELINE" => Trigger.ee? ? nil : "true", # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. + "EE_PIPELINE" => Trigger.ee? ? "true" : nil # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. + } + end + + def version_param_value(_version_file) + raw_version = super + + # if the version matches semver format, treat it as a tag and prepend `v` + if raw_version =~ Regexp.compile(/^\d+\.\d+\.\d+(-rc\d+)?(-ee)?$/) + "v#{raw_version}" + else + raw_version + end + end + end + + class Docs < Base + def self.access_token + # Default to "DOCS_PROJECT_API_TOKEN" at https://gitlab.com/gitlab-org/gitlab-docs/-/settings/access_tokens + ENV['DOCS_PROJECT_API_TOKEN'] || super + end + + SUCCESS_MESSAGE = <<~MSG + => You should now be able to preview your changes under the following URL: + + %<app_url>s + + => For more information, see the documentation + => https://docs.gitlab.com/ee/development/documentation/index.html#previewing-the-changes-live + + => If something doesn't work, drop a line in the #docs chat channel. + MSG + + def deploy! + invoke!.wait! + display_success_message + end + + # + # Remove a remote branch in gitlab-docs. + # + def cleanup! + environment = gitlab_client(:downstream).environments(downstream_project_path, name: downstream_environment).first + return unless environment + + environment = gitlab_client(:downstream).stop_environment(downstream_project_path, environment.id) + if environment.state == 'stopped' + puts "=> Downstream environment '#{downstream_environment}' stopped" + else + puts "=> Downstream environment '#{downstream_environment}' failed to stop." + end + end + + private + + def downstream_environment + "review/#{ref}#{review_slug}" + end + + # We prepend the `-` here because we cannot use variable substitution in `environment.name`/`environment.url` + # Some projects (e.g. `omnibus-gitlab`) use this script for branch pipelines, so we fallback to using `CI_COMMIT_REF_SLUG` for those cases. + def review_slug + identifier = ENV['CI_MERGE_REQUEST_IID'] || ENV['CI_COMMIT_REF_SLUG'] + + "-#{project_slug}-#{identifier}" + end + + def downstream_project_path + ENV.fetch('DOCS_PROJECT_PATH', 'gitlab-org/gitlab-docs') + end + + def ref + ENV.fetch('DOCS_BRANCH', 'main') + end + + # `gitlab-org/gitlab-docs` pipeline trigger "Triggered from gitlab-org/gitlab 'review-docs-deploy' job" + def trigger_token + ENV['DOCS_TRIGGER_TOKEN'] + end + + def extra_variables + { + "BRANCH_#{project_slug.upcase}" => ENV['CI_COMMIT_REF_NAME'], + "REVIEW_SLUG" => review_slug + } + end + + def project_slug + case ENV['CI_PROJECT_PATH'] + when 'gitlab-org/gitlab-foss' + 'ce' + when 'gitlab-org/gitlab' + 'ee' + when 'gitlab-org/gitlab-runner' + 'runner' + when 'gitlab-org/omnibus-gitlab' + 'omnibus' + when 'gitlab-org/charts/gitlab' + 'charts' + end + end + + # app_url is the URL of the `gitlab-docs` Review App URL defined in + # https://gitlab.com/gitlab-org/gitlab-docs/-/blob/b38038132cf82a24271bbb294dead7c2f529e275/.gitlab-ci.yml#L383 + def app_url + "http://#{ref}#{review_slug}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{project_slug}" + end + + def display_success_message + puts format(SUCCESS_MESSAGE, app_url: app_url) + end + end + + class DatabaseTesting < Base + IDENTIFIABLE_NOTE_TAG = 'gitlab-org/database-team/gitlab-com-database-testing:identifiable-note' + + def self.access_token + ENV['GITLABCOM_DATABASE_TESTING_ACCESS_TOKEN'] + end + + def invoke!(post_comment: false, downstream_job_name: nil) + pipeline = super + gitlab = gitlab_client(:upstream) + project_path = base_variables['TOP_UPSTREAM_SOURCE_PROJECT'] + merge_request_id = base_variables['TOP_UPSTREAM_MERGE_REQUEST_IID'] + comment = "<!-- #{IDENTIFIABLE_NOTE_TAG} --> \nStarted database testing [pipeline](https://ops.gitlab.net/#{downstream_project_path}/-/pipelines/#{pipeline.id}) " \ + "(limited access). This comment will be updated once the pipeline has finished running." + + # Look for an existing note + db_testing_notes = gitlab.merge_request_notes(project_path, merge_request_id).auto_paginate.select do |note| + note.body.include?(IDENTIFIABLE_NOTE_TAG) + end + + if db_testing_notes.empty? + # This is the first note + note = gitlab.create_merge_request_note(project_path, merge_request_id, comment) + + puts "Posted comment to:\n" + puts "https://gitlab.com/#{project_path}/-/merge_requests/#{merge_request_id}#note_#{note.id}" + end + end + + private + + def gitlab_client(type) + @gitlab_clients ||= { + downstream: Gitlab.client( + endpoint: 'https://ops.gitlab.net/api/v4', + private_token: self.class.access_token + ), + upstream: Gitlab.client( + endpoint: 'https://gitlab.com/api/v4', + private_token: Base.access_token + ) + } + + @gitlab_clients[type] + end + + def trigger_token + ENV['GITLABCOM_DATABASE_TESTING_TRIGGER_TOKEN'] + end + + def downstream_project_path + ENV.fetch('GITLABCOM_DATABASE_TESTING_PROJECT_PATH', 'gitlab-com/database-team/gitlab-com-database-testing') + end + + def extra_variables + { + # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results + # and fallback to CI_COMMIT_SHA for the `detached` pipelines. + 'GITLAB_COMMIT_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'], + 'TRIGGERED_USER_LOGIN' => ENV['GITLAB_USER_LOGIN'] + } + end + + def ref + ENV['GITLABCOM_DATABASE_TESTING_TRIGGER_REF'] || 'master' + end + end + + class CommitComment + def self.post!(downstream_pipeline, gitlab_client) + gitlab_client.create_commit_comment( + ENV['CI_PROJECT_PATH'], + Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'], + "The [`#{ENV['CI_JOB_NAME']}`](#{ENV['CI_JOB_URL']}) job from pipeline #{ENV['CI_PIPELINE_URL']} triggered #{downstream_pipeline.web_url} downstream.") + + rescue Gitlab::Error::Error => error + puts "Ignoring the following error: #{error}" + end + end + + class Pipeline + INTERVAL = 60 # seconds + MAX_DURATION = 3600 * 3 # 3 hours + + attr_reader :id + + def self.unscoped_class_name + name.split('::').last + end + + def self.gitlab_api_method_name + unscoped_class_name.downcase + end + + def initialize(project, id, gitlab_client) + @project = project + @id = id + @gitlab_client = gitlab_client + @start_time = Time.now.to_i + end + + def wait! + (MAX_DURATION / INTERVAL).times do + case status + when :created, :pending, :running + print "." + sleep INTERVAL + when :success + puts "#{self.class.unscoped_class_name} succeeded in #{duration} minutes!" + return + else + raise "#{self.class.unscoped_class_name} did not succeed!" + end + + $stdout.flush + end + + raise "#{self.class.unscoped_class_name} timed out after waiting for #{duration} minutes!" + end + + def duration + (Time.now.to_i - start_time) / 60 + end + + def status + gitlab_client.public_send(self.class.gitlab_api_method_name, project, id).status.to_sym # rubocop:disable GitlabSecurity/PublicSend + rescue Gitlab::Error::Error => error + puts "Ignoring the following error: #{error}" + # Ignore GitLab API hiccups. If GitLab is really down, we'll hit the job + # timeout anyway. + :running + end + + private + + attr_reader :project, :gitlab_client, :start_time + end + + Job = Class.new(Pipeline) +end + +if $0 == __FILE__ + case ARGV[0] + when 'omnibus' + Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait! + when 'cng' + Trigger::CNG.new.invoke!.wait! + when 'gitlab-com-database-testing' + Trigger::DatabaseTesting.new.invoke! + when 'docs' + docs_trigger = Trigger::Docs.new + + case ARGV[1] + when 'deploy' + docs_trigger.deploy! + when 'cleanup' + docs_trigger.cleanup! + else + puts 'usage: trigger-build docs <deploy|cleanup>' + exit 1 + end + else + puts "Please provide a valid option: + omnibus - Triggers a pipeline that builds the omnibus-gitlab package + cng - Triggers a pipeline that builds images used by the GitLab helm chart + gitlab-com-database-testing - Triggers a pipeline that tests database changes on GitLab.com data" + end +end |