1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
|
#!/usr/bin/env ruby
# frozen_string_literal: true
require 'gitlab'
require_relative 'api/default_options'
# This class allows an upstream job to fetch an artifact from a job in a downstream pipeline.
#
# Until https://gitlab.com/gitlab-org/gitlab/-/issues/285100 is resolved it's not straightforward for an upstream
# pipeline to use artifacts from a downstream pipeline. There is a workaround for parent-child pipelines (see the issue)
# but it relies on CI_MERGE_REQUEST_REF_PATH so it doesn't work for multi-project pipelines.
#
# This uses the Jobs API to get pipeline bridges (trigger jobs) and the Job artifacts API to download artifacts.
# - https://docs.gitlab.com/ee/api/jobs.html#list-pipeline-trigger-jobs
# - https://docs.gitlab.com/ee/api/job_artifacts.html
#
# Note: This class also works for parent-child pipelines within the same project, it's just not necessary in that case.
class DownloadDownstreamArtifact
def initialize(options)
@upstream_project = options.fetch(:upstream_project, API::DEFAULT_OPTIONS[:project])
@upstream_pipeline_id = options.fetch(:upstream_pipeline_id, API::DEFAULT_OPTIONS[:pipeline_id])
@downstream_project = options.fetch(:downstream_project, API::DEFAULT_OPTIONS[:project])
@downstream_job_name = options.fetch(:downstream_job_name)
@trigger_job_name = options.fetch(:trigger_job_name)
@downstream_artifact_path = options.fetch(:downstream_artifact_path)
@output_artifact_path = options.fetch(:output_artifact_path)
unless options.key?(:api_token)
raise ArgumentError, 'PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE is required to access downstream pipelines'
end
api_token = options.fetch(:api_token)
@client = Gitlab.client(
endpoint: options.fetch(:endpoint, API::DEFAULT_OPTIONS[:endpoint]),
private_token: api_token
)
end
def execute
unless downstream_pipeline
abort("Could not find downstream pipeline triggered via #{trigger_job_name} in project #{downstream_project}")
end
unless downstream_job
abort("Could not find job with name '#{downstream_job_name}' in #{downstream_pipeline['web_url']}")
end
puts "Fetching scores artifact from downstream pipeline triggered via #{trigger_job_name}..."
puts "Downstream pipeline is #{downstream_pipeline['web_url']}."
puts %(Downstream job "#{downstream_job_name}": #{downstream_job['web_url']}.)
path = downstream_artifact_path.sub('DOWNSTREAM_JOB_ID', downstream_job.id.to_s)
puts %(Fetching artifact "#{path}" from #{downstream_job_name}...)
download_and_save_artifact(path)
puts "Artifact saved as #{output_artifact_path} ..."
end
def self.options_from_env
API::DEFAULT_OPTIONS.merge({
upstream_project: API::DEFAULT_OPTIONS[:project],
upstream_pipeline_id: API::DEFAULT_OPTIONS[:pipeline_id],
downstream_project: ENV.fetch('DOWNSTREAM_PROJECT', API::DEFAULT_OPTIONS[:project]),
downstream_job_name: ENV['DOWNSTREAM_JOB_NAME'],
trigger_job_name: ENV['TRIGGER_JOB_NAME'],
downstream_artifact_path: ENV['DOWNSTREAM_JOB_ARTIFACT_PATH'],
output_artifact_path: ENV['OUTPUT_ARTIFACT_PATH']
}).except(:project, :pipeline_id)
end
private
attr_reader :downstream_artifact_path,
:output_artifact_path,
:downstream_job_name,
:trigger_job_name,
:upstream_project,
:downstream_project,
:upstream_pipeline_id,
:client
def bridge
@bridge ||= client
.pipeline_bridges(upstream_project, upstream_pipeline_id, per_page: 100)
.auto_paginate
.find { |job| job.name.include?(trigger_job_name) }
end
def downstream_pipeline
@downstream_pipeline ||=
if bridge&.downstream_pipeline.nil?
nil
else
client.pipeline(downstream_project, bridge.downstream_pipeline.id)
end
end
def downstream_job
@downstream_job ||= client
.pipeline_jobs(downstream_project, downstream_pipeline.id)
.find { |job| job.name.include?(downstream_job_name) }
end
def download_and_save_artifact(job_artifact_path)
file_response = client.download_job_artifact_file(downstream_project, downstream_job.id, job_artifact_path)
file_response.respond_to?(:read) || abort("Could not download artifact. Request returned: #{file_response}")
File.write(output_artifact_path, file_response.read)
end
end
if $PROGRAM_NAME == __FILE__
options = DownloadDownstreamArtifact.options_from_env
DownloadDownstreamArtifact.new(options).execute
end
|
