diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-12-14 12:07:51 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-12-14 12:07:51 +0300 |
commit | 00b8ecb72c9f77d864aff3572f028613f45af03c (patch) | |
tree | cb8f42234547d61f2721e3fe9361e84c6a710235 | |
parent | 138c61238317b2a61f387749a1f4583309675a83 (diff) |
Add latest changes from gitlab-org/gitlab@master
-rw-r--r-- | app/models/ci/pipeline_enums.rb | 3 | ||||
-rw-r--r-- | app/services/ci/create_pipeline_service.rb | 1 | ||||
-rw-r--r-- | doc/administration/external_pipeline_validation.md | 103 | ||||
-rw-r--r-- | doc/administration/index.md | 1 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/helpers.rb | 5 | ||||
-rw-r--r-- | lib/gitlab/ci/pipeline/chain/validate/external.rb | 100 | ||||
-rw-r--r-- | spec/fixtures/api/schemas/external_validation.json | 75 | ||||
-rw-r--r-- | spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb | 103 |
8 files changed, 388 insertions, 3 deletions
diff --git a/app/models/ci/pipeline_enums.rb b/app/models/ci/pipeline_enums.rb index ac930f63abf..3cd88807969 100644 --- a/app/models/ci/pipeline_enums.rb +++ b/app/models/ci/pipeline_enums.rb @@ -7,7 +7,8 @@ module Ci def self.failure_reasons { unknown_failure: 0, - config_error: 1 + config_error: 1, + external_validation_failure: 2 } end diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index 8887534bece..ce3a9eb0772 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -16,6 +16,7 @@ module Ci Gitlab::Ci::Pipeline::Chain::EvaluateWorkflowRules, Gitlab::Ci::Pipeline::Chain::Seed, Gitlab::Ci::Pipeline::Chain::Limit::Size, + Gitlab::Ci::Pipeline::Chain::Validate::External, Gitlab::Ci::Pipeline::Chain::Populate, Gitlab::Ci::Pipeline::Chain::Create, Gitlab::Ci::Pipeline::Chain::Limit::Activity, diff --git a/doc/administration/external_pipeline_validation.md b/doc/administration/external_pipeline_validation.md new file mode 100644 index 00000000000..19d4de3b705 --- /dev/null +++ b/doc/administration/external_pipeline_validation.md @@ -0,0 +1,103 @@ +# External Pipeline Validation + +You can use an external service for validating a pipeline before it's created. + +CAUTION: **Warning:** +This is an experimental feature and subject to change without notice. + +## Usage + +GitLab will send a POST request to the external service URL with the pipeline +data as payload. GitLab will then invalidate the pipeline based on the response +code. If there's an error or the request times out, the pipeline will not be +invalidated. + +Response Code Legend: + +- `200` - Accepted +- `4xx` - Not Accepted +- Other Codes - Accepted and Logged + +## Configuration + +Set the `EXTERNAL_VALIDATION_SERVICE_URL` to the external service url. + +## Payload Schema + +```json +{ + "type": "object", + "required" : [ + "project", + "user", + "pipeline", + "builds" + ], + "properties" : { + "project": { + "type": "object", + "required": [ + "id", + "path" + ], + "properties": { + "id": { "type": "integer" }, + "path": { "type": "string" } + } + }, + "user": { + "type": "object", + "required": [ + "id", + "username", + "email" + ], + "properties": { + "id": { "type": "integer" }, + "username": { "type": "string" }, + "email": { "type": "string" } + } + }, + "pipeline": { + "type": "object", + "required": [ + "sha", + "ref", + "type" + ], + "properties": { + "sha": { "type": "string" }, + "ref": { "type": "string" }, + "type": { "type": "string" } + } + }, + "builds": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "stage", + "image", + "services", + "script" + ], + "properties": { + "name": { "type": "string" }, + "stage": { "type": "string" }, + "image": { "type": ["string", "null"] }, + "services": { + "type": ["array", "null"], + "items": { "type": "string" } + }, + "script": { + "type": "array", + "items": { "type": "string" } + } + } + } + } + }, + "additionalProperties": false +} +``` diff --git a/doc/administration/index.md b/doc/administration/index.md index 40ec9b85455..1652b287258 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -152,6 +152,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Enable/disable GitLab CI/CD](../ci/enable_or_disable_ci.md#site-wide-admin-setting): Enable or disable GitLab CI/CD for your instance. - [GitLab CI/CD admin settings](../user/admin_area/settings/continuous_integration.md): Enable or disable Auto DevOps site-wide and define the artifacts' max size and expiration time. +- [External Pipeline Validation](external_pipeline_validation.md): Enable, disable and configure external pipeline validation. - [Job artifacts](job_artifacts.md): Enable, disable, and configure job artifacts (a set of files and directories which are outputted by a job when it completes successfully). - [Job logs](job_logs.md): Information about the job logs. - [Register Shared and specific Runners](../ci/runners/README.md#registering-a-shared-runner): Learn how to register and configure Shared and specific Runners to your own instance. diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index 8ccb1066575..982ecc0ff51 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -5,12 +5,13 @@ module Gitlab module Pipeline module Chain module Helpers - def error(message, config_error: false) + def error(message, config_error: false, drop_reason: nil) if config_error && command.save_incompleted + drop_reason = :config_error pipeline.yaml_errors = message - pipeline.drop!(:config_error) end + pipeline.drop!(drop_reason) if drop_reason pipeline.errors.add(:base, message) end end diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb new file mode 100644 index 00000000000..97af42b5fd6 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Validate + class External < Chain::Base + include Chain::Helpers + + InvalidResponseCode = Class.new(StandardError) + + VALIDATION_REQUEST_TIMEOUT = 5 + + def perform! + error('External validation failed', drop_reason: :external_validation_failure) unless validate_external + end + + def break? + @pipeline.errors.any? + end + + private + + def validate_external + return true unless validation_service_url + + # 200 - accepted + # 4xx - not accepted + # everything else - accepted and logged + response_code = validate_service_request.code + case response_code + when 200 + true + when 400..499 + false + else + raise InvalidResponseCode, "Unsupported response code received from Validation Service: #{response_code}" + end + rescue => ex + Gitlab::Sentry.track_exception(ex) + + true + end + + def validate_service_request + Gitlab::HTTP.post( + validation_service_url, timeout: VALIDATION_REQUEST_TIMEOUT, + body: validation_service_payload(@pipeline, @command.config_processor.stages_attributes) + ) + end + + def validation_service_url + ENV['EXTERNAL_VALIDATION_SERVICE_URL'] + end + + def validation_service_payload(pipeline, stages_attributes) + { + project: { + id: pipeline.project.id, + path: pipeline.project.full_path + }, + user: { + id: pipeline.user.id, + username: pipeline.user.username, + email: pipeline.user.email + }, + pipeline: { + sha: pipeline.sha, + ref: pipeline.ref, + type: pipeline.source + }, + builds: builds_validation_payload(stages_attributes) + }.to_json + end + + def builds_validation_payload(stages_attributes) + stages_attributes.map { |stage| stage[:builds] }.flatten + .map(&method(:build_validation_payload)) + end + + def build_validation_payload(build) + { + name: build[:name], + stage: build[:stage], + image: build.dig(:options, :image, :name), + services: build.dig(:options, :services)&.map { |service| service[:name] }, + script: [ + build.dig(:options, :before_script), + build.dig(:options, :script), + build.dig(:options, :after_script) + ].flatten.compact + } + end + end + end + end + end + end +end diff --git a/spec/fixtures/api/schemas/external_validation.json b/spec/fixtures/api/schemas/external_validation.json new file mode 100644 index 00000000000..1bd00a2e6fc --- /dev/null +++ b/spec/fixtures/api/schemas/external_validation.json @@ -0,0 +1,75 @@ +{ + "type": "object", + "required" : [ + "project", + "user", + "pipeline", + "builds" + ], + "properties" : { + "project": { + "type": "object", + "required": [ + "id", + "path" + ], + "properties": { + "id": { "type": "integer" }, + "path": { "type": "string" } + } + }, + "user": { + "type": "object", + "required": [ + "id", + "username", + "email" + ], + "properties": { + "id": { "type": "integer" }, + "username": { "type": "string" }, + "email": { "type": "string" } + } + }, + "pipeline": { + "type": "object", + "required": [ + "sha", + "ref", + "type" + ], + "properties": { + "sha": { "type": "string" }, + "ref": { "type": "string" }, + "type": { "type": "string" } + } + }, + "builds": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "stage", + "image", + "services", + "script" + ], + "properties": { + "name": { "type": "string" }, + "stage": { "type": "string" }, + "image": { "type": ["string", "null"] }, + "services": { + "type": ["array", "null"], + "items": { "type": "string" } + }, + "script": { + "type": "array", + "items": { "type": "string" } + } + } + } + } + }, + "additionalProperties": false +} diff --git a/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb b/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb new file mode 100644 index 00000000000..f2a0b93ef28 --- /dev/null +++ b/spec/lib/gitlab/ci/pipeline/chain/validate/external_spec.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Ci::Pipeline::Chain::Validate::External do + let(:project) { create(:project) } + let(:user) { create(:user) } + let(:pipeline) { build(:ci_empty_pipeline, user: user, project: project) } + let!(:step) { described_class.new(pipeline, command) } + + let(:ci_yaml) do + <<-CI_YAML + stages: + - first_stage + - second_stage + + first_stage_job_name: + stage: first_stage + image: hello_world + script: + - echo 'hello' + + second_stage_job_name: + stage: second_stage + services: + - postgres + before_script: + - echo 'first hello' + script: + - echo 'second hello' + CI_YAML + end + + let(:yaml_processor) do + ::Gitlab::Ci::YamlProcessor.new( + ci_yaml, { + project: project, + sha: pipeline.sha, + user: user + } + ) + end + + let(:command) do + Gitlab::Ci::Pipeline::Chain::Command.new( + project: project, current_user: user, config_processor: yaml_processor + ) + end + + describe '#perform!' do + subject(:perform!) { step.perform! } + + context 'when validation returns true' do + before do + allow(step).to receive(:validate_external).and_return(true) + end + + it 'does not drop the pipeline' do + perform! + + expect(pipeline.status).not_to eq('failed') + expect(pipeline.errors).to be_empty + end + + it 'does not break the chain' do + perform! + + expect(step.break?).to be false + end + end + + context 'when validation return false' do + before do + allow(step).to receive(:validate_external).and_return(false) + end + + it 'drops the pipeline' do + perform! + + expect(pipeline.status).to eq('failed') + expect(pipeline.errors.to_a).to include('External validation failed') + end + + it 'breaks the chain' do + perform! + + expect(step.break?).to be true + end + end + end + + describe '#validation_service_payload' do + subject(:validation_service_payload) { step.send(:validation_service_payload, pipeline, command.config_processor.stages_attributes) } + + it 'respects the defined schema' do + expect(validation_service_payload).to match_schema('/external_validation') + end + + it 'does not fire sql queries' do + expect { validation_service_payload }.not_to exceed_query_limit(1) + end + end +end |