diff options
author | Kamil Trzciński <ayufan@ayufan.eu> | 2017-09-04 15:30:27 +0300 |
---|---|---|
committer | Kamil Trzciński <ayufan@ayufan.eu> | 2017-09-04 15:30:27 +0300 |
commit | a0c13698f948d4ebc69ce54660c4022a4a394a0c (patch) | |
tree | 1f74e480077a51862cbed0ee8df9de35ad22b09c | |
parent | 794a2630fdfa335d90bff0e7e2f37a4b13145ee7 (diff) | |
parent | e23e86953de26b1681cbde5b3847e631e5ae7c74 (diff) |
Merge branch 'feature/gb/kubernetes-only-pipeline-jobs' into 'master'
Check if Kubernetes is required when creating pipeline jobs
Closes #34785
See merge request !13849
-rw-r--r-- | app/models/ci/pipeline.rb | 4 | ||||
-rw-r--r-- | app/views/ci/lints/_create.html.haml | 4 | ||||
-rw-r--r-- | changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml | 5 | ||||
-rw-r--r-- | doc/ci/yaml/README.md | 38 | ||||
-rw-r--r-- | lib/ci/gitlab_ci_yaml_processor.rb | 49 | ||||
-rw-r--r-- | lib/gitlab/ci/config/entry/policy.rb | 22 | ||||
-rw-r--r-- | lib/gitlab/ci/config/entry/validators.rb | 8 | ||||
-rw-r--r-- | spec/lib/ci/gitlab_ci_yaml_processor_spec.rb | 39 | ||||
-rw-r--r-- | spec/lib/gitlab/ci/config/entry/policy_spec.rb | 48 | ||||
-rw-r--r-- | spec/models/ci/pipeline_spec.rb | 16 | ||||
-rw-r--r-- | spec/models/project_services/kubernetes_service_spec.rb | 13 | ||||
-rw-r--r-- | spec/views/ci/lints/show.html.haml_spec.rb | 4 |
12 files changed, 210 insertions, 40 deletions
diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index ca9a350ea79..35d14b6e297 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -305,6 +305,10 @@ module Ci @stage_seeds ||= config_processor.stage_seeds(self) end + def has_kubernetes_active? + project.kubernetes_service&.active? + end + def has_stage_seeds? stage_seeds.any? end diff --git a/app/views/ci/lints/_create.html.haml b/app/views/ci/lints/_create.html.haml index c91602fcff7..30bf1384b22 100644 --- a/app/views/ci/lints/_create.html.haml +++ b/app/views/ci/lints/_create.html.haml @@ -22,10 +22,10 @@ %b Tag list: = build[:tag_list].to_a.join(", ") %br - %b Refs only: + %b Only policy: = @jobs[build[:name].to_sym][:only].to_a.join(", ") %br - %b Refs except: + %b Except policy: = @jobs[build[:name].to_sym][:except].to_a.join(", ") %br %b Environment: diff --git a/changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml b/changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml new file mode 100644 index 00000000000..00c38a0c671 --- /dev/null +++ b/changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml @@ -0,0 +1,5 @@ +--- +title: Add CI/CD active kubernetes job policy +merge_request: 13849 +author: +type: added diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index abf4ec7dbf8..cacfd2ed254 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -427,16 +427,16 @@ a "key: value" pair. Be careful when using special characters: are executed in `parallel`. For more info about the use of `stage` please check [stages](#stages). -### only and except +### only and except (simplified) -`only` and `except` are two parameters that set a refs policy to limit when -jobs are built: +`only` and `except` are two parameters that set a job policy to limit when +jobs are created: 1. `only` defines the names of branches and tags for which the job will run. 2. `except` defines the names of branches and tags for which the job will **not** run. -There are a few rules that apply to the usage of refs policy: +There are a few rules that apply to the usage of job policy: * `only` and `except` are inclusive. If both `only` and `except` are defined in a job specification, the ref is filtered by `only` and `except`. @@ -497,6 +497,36 @@ job: The above example will run `job` for all branches on `gitlab-org/gitlab-ce`, except master. +### only and except (complex) + +> Introduced in GitLab 10.0 + +> This an _alpha_ feature, and it it subject to change at any time without + prior notice! + +Since GitLab 10.0 it is possible to define a more elaborate only/except job +policy configuration. + +GitLab now supports both, simple and complex strategies, so it is possible to +use an array and a hash configuration scheme. + +Two keys are now available: `refs` and `kubernetes`. Refs strategy equals to +simplified only/except configuration, whereas kubernetes strategy accepts only +`active` keyword. + +See the example below. Job is going to be created only when pipeline has been +scheduled or runs for a `master` branch, and only if kubernetes service is +active in the project. + +```yaml +job: + only: + refs: + - master + - schedules + kubernetes: active +``` + ### Job variables It is possible to define job variables using a `variables` keyword on a job diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 3a4911b23b0..62b44389b15 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -20,24 +20,6 @@ module Ci raise ValidationError, e.message end - def jobs_for_ref(ref, tag = false, source = nil) - @jobs.select do |_, job| - process?(job[:only], job[:except], ref, tag, source) - end - end - - def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).select do |_, job| - job[:stage] == stage - end - end - - def builds_for_ref(ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).map do |name, _| - build_attributes(name) - end - end - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| build_attributes(name) @@ -52,8 +34,7 @@ module Ci def stage_seeds(pipeline) seeds = @stages.uniq.map do |stage| - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) + builds = pipeline_stage_builds(stage, pipeline) Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? end @@ -101,6 +82,34 @@ module Ci private + def pipeline_stage_builds(stage, pipeline) + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, pipeline.source) + + builds.select do |build| + job = @jobs[build.fetch(:name).to_sym] + has_kubernetes = pipeline.has_kubernetes_active? + only_kubernetes = job.dig(:only, :kubernetes) + except_kubernetes = job.dig(:except, :kubernetes) + + [!only_kubernetes && !except_kubernetes, + only_kubernetes && has_kubernetes, + except_kubernetes && !has_kubernetes].any? + end + end + + def jobs_for_ref(ref, tag = false, source = nil) + @jobs.select do |_, job| + process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) + end + end + + def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) + jobs_for_ref(ref, tag, source).select do |_, job| + job[:stage] == stage + end + end + def initial_parsing ## # Global config diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 3cdae1cee4f..0027e9ec8c5 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -7,6 +7,7 @@ module Gitlab # class Policy < Simplifiable strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } + strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) } class RefsPolicy < Entry::Node include Entry::Validatable @@ -14,6 +15,27 @@ module Gitlab validations do validates :config, array_of_strings_or_regexps: true end + + def value + { refs: @config } + end + end + + class ComplexPolicy < Entry::Node + include Entry::Validatable + include Entry::Attributable + + attributes :refs, :kubernetes + + validations do + validates :config, presence: true + validates :config, allowed_keys: %i[refs kubernetes] + + with_options allow_nil: true do + validates :refs, array_of_strings_or_regexps: true + validates :kubernetes, allowed_values: %w[active] + end + end end class UnknownStrategy < Entry::Node diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index b2ca3c881e4..0159179f0a9 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -14,6 +14,14 @@ module Gitlab end end + class AllowedValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless options[:in].include?(value.to_s) + record.errors.add(attribute, "unknown value: #{value}") + end + end + end + class ArrayOfStringsValidator < ActiveModel::EachValidator include LegacyValidationHelpers diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index c70a4cb55fe..1efd3113a43 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -164,9 +164,46 @@ module Ci expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' end end + + context 'when kubernetes policy is specified' do + let(:pipeline) { create(:ci_empty_pipeline) } + + let(:config) do + YAML.dump( + spinach: { stage: 'test', script: 'spinach' }, + production: { + stage: 'deploy', + script: 'cap', + only: { kubernetes: 'active' } + } + ) + end + + context 'when kubernetes is active' do + let(:project) { create(:kubernetes_project) } + let(:pipeline) { create(:ci_empty_pipeline, project: project) } + + it 'returns seeds for kubernetes dependent job' do + seeds = subject.stage_seeds(pipeline) + + expect(seeds.size).to eq 2 + expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' + expect(seeds.second.builds.dig(0, :name)).to eq 'production' + end + end + + context 'when kubernetes is not active' do + it 'does not return seeds for kubernetes dependent job' do + seeds = subject.stage_seeds(pipeline) + + expect(seeds.size).to eq 1 + expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' + end + end + end end - describe "#builds_for_ref" do + describe "#builds_for_stage_and_ref" do let(:type) { 'test' } it "returns builds if no branch specified" do diff --git a/spec/lib/gitlab/ci/config/entry/policy_spec.rb b/spec/lib/gitlab/ci/config/entry/policy_spec.rb index 36a84da4a52..5e83abf645b 100644 --- a/spec/lib/gitlab/ci/config/entry/policy_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/policy_spec.rb @@ -16,8 +16,8 @@ describe Gitlab::Ci::Config::Entry::Policy do end describe '#value' do - it 'returns key value' do - expect(entry.value).to eq config + it 'returns refs hash' do + expect(entry.value).to eq(refs: config) end end end @@ -56,6 +56,50 @@ describe Gitlab::Ci::Config::Entry::Policy do end end + context 'when using complex policy' do + context 'when specifiying refs policy' do + let(:config) { { refs: ['master'] } } + + it 'is a correct configuraton' do + expect(entry).to be_valid + expect(entry.value).to eq(refs: %w[master]) + end + end + + context 'when specifying kubernetes policy' do + let(:config) { { kubernetes: 'active' } } + + it 'is a correct configuraton' do + expect(entry).to be_valid + expect(entry.value).to eq(kubernetes: 'active') + end + end + + context 'when specifying invalid kubernetes policy' do + let(:config) { { kubernetes: 'something' } } + + it 'reports an error about invalid policy' do + expect(entry.errors).to include /unknown value: something/ + end + end + + context 'when specifying unknown policy' do + let(:config) { { refs: ['master'], invalid: :something } } + + it 'returns error about invalid key' do + expect(entry.errors).to include /unknown keys: invalid/ + end + end + + context 'when policy is empty' do + let(:config) { {} } + + it 'is not a valid configuration' do + expect(entry.errors).to include /can't be blank/ + end + end + end + context 'when policy strategy does not match' do let(:config) { 'string strategy' } diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index b84e3ff18e8..84656ffe0b9 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -546,6 +546,22 @@ describe Ci::Pipeline, :mailer do end end + describe '#has_kubernetes_active?' do + context 'when kubernetes is active' do + let(:project) { create(:kubernetes_project) } + + it 'returns true' do + expect(pipeline).to have_kubernetes_active + end + end + + context 'when kubernetes is not active' do + it 'returns false' do + expect(pipeline).not_to have_kubernetes_active + end + end + end + describe '#has_stage_seeds?' do context 'when pipeline has stage seeds' do subject { build(:ci_pipeline_with_one_job) } diff --git a/spec/models/project_services/kubernetes_service_spec.rb b/spec/models/project_services/kubernetes_service_spec.rb index b1743cd608e..537cdadd528 100644 --- a/spec/models/project_services/kubernetes_service_spec.rb +++ b/spec/models/project_services/kubernetes_service_spec.rb @@ -203,18 +203,13 @@ describe KubernetesService, :use_clean_rails_memory_store_caching do describe '#predefined_variables' do let(:kubeconfig) do - config = - YAML.load(File.read(expand_fixture_path('config/kubeconfig.yml'))) - - config.dig('users', 0, 'user')['token'] = - 'token' - + config_file = expand_fixture_path('config/kubeconfig.yml') + config = YAML.load(File.read(config_file)) + config.dig('users', 0, 'user')['token'] = 'token' + config.dig('contexts', 0, 'context')['namespace'] = namespace config.dig('clusters', 0, 'cluster')['certificate-authority-data'] = Base64.encode64('CA PEM DATA') - config.dig('contexts', 0, 'context')['namespace'] = - namespace - YAML.dump(config) end diff --git a/spec/views/ci/lints/show.html.haml_spec.rb b/spec/views/ci/lints/show.html.haml_spec.rb index 3390ae247ff..f2c19c7642a 100644 --- a/spec/views/ci/lints/show.html.haml_spec.rb +++ b/spec/views/ci/lints/show.html.haml_spec.rb @@ -73,8 +73,8 @@ describe 'ci/lints/show' do render expect(rendered).to have_content('Tag list: dotnet') - expect(rendered).to have_content('Refs only: test@dude/repo') - expect(rendered).to have_content('Refs except: deploy') + expect(rendered).to have_content('Only policy: refs, test@dude/repo') + expect(rendered).to have_content('Except policy: refs, deploy') expect(rendered).to have_content('Environment: testing') expect(rendered).to have_content('When: on_success') end |