diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/api/entities.rb | 78 | ||||
| -rw-r--r-- | lib/api/helpers/runner.rb | 54 | ||||
| -rw-r--r-- | lib/api/runner.rb | 198 | ||||
| -rw-r--r-- | lib/ci/api/builds.rb | 2 | ||||
| -rw-r--r-- | lib/gitlab/ci/build/image.rb | 33 | ||||
| -rw-r--r-- | lib/gitlab/ci/build/step.rb | 46 |
6 files changed, 410 insertions, 1 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 2230aa0706b..c8f21fc9ca8 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -705,5 +705,83 @@ module API expose :id, :message, :starts_at, :ends_at, :color, :font expose :active?, as: :active end + + module JobRequest + class JobInfo < Grape::Entity + expose :name, :stage + expose :project_id, :project_name + end + + class GitInfo < Grape::Entity + expose :repo_url, :ref, :sha, :before_sha + expose :ref_type do |model| + if model.tag + 'tag' + else + 'branch' + end + end + end + + class RunnerInfo < Grape::Entity + expose :timeout + end + + class Step < Grape::Entity + expose :name, :script, :timeout, :when, :allow_failure + end + + class Image < Grape::Entity + expose :name + end + + class Artifacts < Grape::Entity + expose :name, :untracked, :paths, :when, :expire_in + end + + class Cache < Grape::Entity + expose :key, :untracked, :paths + end + + class Credentials < Grape::Entity + expose :type, :url, :username, :password + end + + class ArtifactFile < Grape::Entity + expose :filename, :size + end + + class Dependency < Grape::Entity + expose :id, :name + expose :artifacts_file, using: ArtifactFile, if: ->(job, _) { job.artifacts? } + end + + class Response < Grape::Entity + expose :id + expose :token + expose :allow_git_fetch + + expose :job_info, using: JobInfo do |model| + model + end + + expose :git_info, using: GitInfo do |model| + model + end + + expose :runner_info, using: RunnerInfo do |model| + model + end + + expose :variables + expose :steps, using: Step + expose :image, using: Image + expose :services, using: Image + expose :artifacts, using: Artifacts + expose :cache, using: Cache + expose :credentials, using: Credentials + expose :depends_on_builds, as: :dependencies, using: Dependency + end + end end end diff --git a/lib/api/helpers/runner.rb b/lib/api/helpers/runner.rb index 119ca81b883..ec2bcaed929 100644 --- a/lib/api/helpers/runner.rb +++ b/lib/api/helpers/runner.rb @@ -1,6 +1,10 @@ module API module Helpers module Runner + JOB_TOKEN_HEADER = 'HTTP_JOB_TOKEN'.freeze + JOB_TOKEN_PARAM = :token + UPDATE_RUNNER_EVERY = 10 * 60 + def runner_registration_token_valid? ActiveSupport::SecurityUtils.variable_size_secure_compare(params[:token], current_application_settings.runners_registration_token) @@ -18,6 +22,56 @@ module API def current_runner @runner ||= ::Ci::Runner.find_by_token(params[:token].to_s) end + + def update_runner_info + return unless update_runner? + + current_runner.contacted_at = Time.now + current_runner.assign_attributes(get_runner_version_from_params) + current_runner.save if current_runner.changed? + end + + def update_runner? + # Use a random threshold to prevent beating DB updates. + # It generates a distribution between [40m, 80m]. + # + contacted_at_max_age = UPDATE_RUNNER_EVERY + Random.rand(UPDATE_RUNNER_EVERY) + + current_runner.contacted_at.nil? || + (Time.now - current_runner.contacted_at) >= contacted_at_max_age + end + + def job_not_found! + if headers['User-Agent'].to_s =~ /gitlab(-ci-multi)?-runner \d+\.\d+\.\d+(~beta\.\d+\.g[0-9a-f]+)? / + no_content! + else + not_found! + end + end + + def validate_job!(job) + not_found! unless job + + yield if block_given? + + forbidden!('Project has been deleted!') unless job.project + forbidden!('Job has been erased!') if job.erased? + end + + def authenticate_job!(job) + validate_job!(job) do + forbidden! unless job_token_valid?(job) + end + end + + def job_token_valid?(job) + token = (params[JOB_TOKEN_PARAM] || env[JOB_TOKEN_HEADER]).to_s + token && job.valid_token?(token) + end + + def max_artifacts_size + current_application_settings.max_artifacts_size.megabytes.to_i + end end end end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 47858f1866b..c700d2ef4a1 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -48,5 +48,203 @@ module API Ci::Runner.find_by_token(params[:token]).destroy end end + + resource :jobs do + desc 'Request a job' do + success Entities::JobRequest::Response + end + params do + requires :token, type: String, desc: %q(Runner's authentication token) + optional :last_update, type: String, desc: %q(Runner's queue last_update token) + optional :info, type: Hash, desc: %q(Runner's metadata) + end + post '/request' do + authenticate_runner! + not_found! unless current_runner.active? + update_runner_info + + if current_runner.is_runner_queue_value_latest?(params[:last_update]) + header 'X-GitLab-Last-Update', params[:last_update] + Gitlab::Metrics.add_event(:build_not_found_cached) + return job_not_found! + end + + new_update = current_runner.ensure_runner_queue_value + result = ::Ci::RegisterJobService.new(current_runner).execute + + if result.valid? + if result.build + Gitlab::Metrics.add_event(:build_found, + project: result.build.project.path_with_namespace) + present result.build, with: Entities::JobRequest::Response + else + Gitlab::Metrics.add_event(:build_not_found) + header 'X-GitLab-Last-Update', new_update + job_not_found! + end + else + # We received build that is invalid due to concurrency conflict + Gitlab::Metrics.add_event(:build_invalid) + conflict! + end + end + + desc 'Updates a job' do + http_codes [[200, 'Job was updated'], [403, 'Forbidden']] + end + params do + requires :token, type: String, desc: %q(Runners's authentication token) + requires :id, type: Integer, desc: %q(Job's ID) + optional :trace, type: String, desc: %q(Job's full trace) + optional :state, type: String, desc: %q(Job's status: success, failed) + end + put '/:id' do + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + + job.update_attributes(trace: params[:trace]) if params[:trace] + + Gitlab::Metrics.add_event(:update_build, + project: job.project.path_with_namespace) + + case params[:state].to_s + when 'success' + job.success + when 'failed' + job.drop + end + end + + desc 'Appends a patch to the job trace' do + http_codes [[202, 'Trace was patched'], + [400, 'Missing Content-Range header'], + [403, 'Forbidden'], + [416, 'Range not satisfiable']] + end + params do + requires :id, type: Integer, desc: %q(Job's ID) + optional :token, type: String, desc: %q(Job's authentication token) + end + patch '/:id/trace' do + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + + error!('400 Missing header Content-Range', 400) unless request.headers.has_key?('Content-Range') + content_range = request.headers['Content-Range'] + content_range = content_range.split('-') + + current_length = job.trace_length + unless current_length == content_range[0].to_i + return error!('416 Range Not Satisfiable', 416, { 'Range' => "0-#{current_length}" }) + end + + job.append_trace(request.body.read, content_range[0].to_i) + + status 202 + header 'Job-Status', job.status + header 'Range', "0-#{job.trace_length}" + end + + desc 'Authorize artifacts uploading for job' do + http_codes [[200, 'Upload allowed'], + [403, 'Forbidden'], + [405, 'Artifacts support not enabled'], + [413, 'File too large']] + end + params do + requires :id, type: Integer, desc: %q(Job's ID) + optional :token, type: String, desc: %q(Job's authentication token) + optional :filesize, type: Integer, desc: %q(Artifacts filesize) + end + post '/:id/artifacts/authorize' do + not_allowed! unless Gitlab.config.artifacts.enabled + require_gitlab_workhorse! + Gitlab::Workhorse.verify_api_request!(headers) + + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + forbidden!('Job is not running') unless job.running? + + if params[:filesize] + file_size = params[:filesize].to_i + file_to_large! unless file_size < max_artifacts_size + end + + status 200 + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + Gitlab::Workhorse.artifact_upload_ok + end + + desc 'Upload artifacts for job' do + success Entities::JobRequest::Response + http_codes [[201, 'Artifact uploaded'], + [400, 'Bad request'], + [403, 'Forbidden'], + [405, 'Artifacts support not enabled'], + [413, 'File too large']] + end + params do + requires :id, type: Integer, desc: %q(Job's ID) + optional :token, type: String, desc: %q(Job's authentication token) + optional :expire_in, type: String, desc: %q(Specify when artifacts should expire) + optional :file, type: File, desc: %q(Artifact's file) + optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) + optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) + optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) + optional 'metadata.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) + optional 'metadata.name', type: String, desc: %q(filename (generated by Workhorse)) + end + post '/:id/artifacts' do + not_allowed! unless Gitlab.config.artifacts.enabled + require_gitlab_workhorse! + + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + forbidden!('Job is not running!') unless job.running? + + artifacts_upload_path = ArtifactUploader.artifacts_upload_path + artifacts = uploaded_file(:file, artifacts_upload_path) + metadata = uploaded_file(:metadata, artifacts_upload_path) + + bad_request!('Missing artifacts file!') unless artifacts + file_to_large! unless artifacts.size < max_artifacts_size + + job.artifacts_file = artifacts + job.artifacts_metadata = metadata + job.artifacts_expire_in = params['expire_in'] || + Gitlab::CurrentSettings.current_application_settings.default_artifacts_expire_in + + if job.save + present job, with: Entities::JobRequest::Response + else + render_validation_error!(job) + end + end + + desc 'Download the artifacts file for job' do + http_codes [[200, 'Upload allowed'], + [403, 'Forbidden'], + [404, 'Artifact not found']] + end + params do + requires :id, type: Integer, desc: %q(Job's ID) + optional :token, type: String, desc: %q(Job's authentication token) + end + get '/:id/artifacts' do + job = Ci::Build.find_by_id(params[:id]) + authenticate_job!(job) + + artifacts_file = job.artifacts_file + unless artifacts_file.file_storage? + return redirect_to job.artifacts_file.url + end + + unless artifacts_file.exists? + not_found! + end + + present_file!(artifacts_file.path, artifacts_file.filename) + end + end end end diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index b51e76d93f2..746e76a1b1f 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -24,7 +24,7 @@ module Ci new_update = current_runner.ensure_runner_queue_value - result = Ci::RegisterBuildService.new(current_runner).execute + result = Ci::RegisterJobService.new(current_runner).execute if result.valid? if result.build diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb new file mode 100644 index 00000000000..c62aeb60fa9 --- /dev/null +++ b/lib/gitlab/ci/build/image.rb @@ -0,0 +1,33 @@ +module Gitlab + module Ci + module Build + class Image + attr_reader :name + + class << self + def from_image(job) + image = Gitlab::Ci::Build::Image.new(job.options[:image]) + return unless image.valid? + image + end + + def from_services(job) + services = job.options[:services].to_a.map do |service| + Gitlab::Ci::Build::Image.new(service) + end + + services.select(&:valid?).compact + end + end + + def initialize(image) + @name = image + end + + def valid? + @name.present? + end + end + end + end +end diff --git a/lib/gitlab/ci/build/step.rb b/lib/gitlab/ci/build/step.rb new file mode 100644 index 00000000000..1877429ac46 --- /dev/null +++ b/lib/gitlab/ci/build/step.rb @@ -0,0 +1,46 @@ +module Gitlab + module Ci + module Build + class Step + WHEN_ON_FAILURE = 'on_failure'.freeze + WHEN_ON_SUCCESS = 'on_success'.freeze + WHEN_ALWAYS = 'always'.freeze + + attr_reader :name + attr_writer :script + attr_accessor :timeout, :when, :allow_failure + + class << self + def from_commands(job) + self.new(:script).tap do |step| + step.script = job.commands + step.timeout = job.timeout + step.when = WHEN_ON_SUCCESS + end + end + + def from_after_script(job) + after_script = job.options[:after_script] + return unless after_script + + self.new(:after_script).tap do |step| + step.script = after_script + step.timeout = job.timeout + step.when = WHEN_ALWAYS + step.allow_failure = true + end + end + end + + def initialize(name) + @name = name + @allow_failure = false + end + + def script + @script.split("\n") + end + end + end + end +end |