From 5ce9e03f6c015ab28aac1b3490acbf6f1c6f538f Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 26 Aug 2017 11:11:28 +0200 Subject: Use new complex only/except policy internal scheme --- lib/ci/gitlab_ci_yaml_processor.rb | 2 +- lib/gitlab/ci/config/entry/policy.rb | 4 ++++ spec/lib/gitlab/ci/config/entry/policy_spec.rb | 4 ++-- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 3a4911b23b0..3efd9b3bdac 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -22,7 +22,7 @@ module Ci def jobs_for_ref(ref, tag = false, source = nil) @jobs.select do |_, job| - process?(job[:only], job[:except], ref, tag, source) + process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) end end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 05602f1b3c6..bcb76de65b2 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -15,6 +15,10 @@ module Gitlab validations do validates :config, array_of_strings_or_regexps: true end + + def value + { refs: @config } + end end class ExpressionsPolicy < Entry::Node diff --git a/spec/lib/gitlab/ci/config/entry/policy_spec.rb b/spec/lib/gitlab/ci/config/entry/policy_spec.rb index ccf885969fb..a7dae11cee7 100644 --- a/spec/lib/gitlab/ci/config/entry/policy_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/policy_spec.rb @@ -16,8 +16,8 @@ describe Gitlab::Ci::Config::Entry::Policy do end describe '#value' do - it 'returns key value' do - expect(entry.value).to eq config + it 'returns refs hash' do + expect(entry.value).to eq(refs: config) end end end -- cgit v1.2.3 From cdcc196c1fb602303a4fae36a584005ef212b9c8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 26 Aug 2017 11:17:47 +0200 Subject: Make it possible to access deployment details from pipeline --- app/models/ci/pipeline.rb | 1 + spec/models/ci/pipeline_spec.rb | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index ea7331cb27f..36381956195 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -31,6 +31,7 @@ module Ci has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id' delegate :id, to: :project, prefix: true + delegate :deployment_variables, to: :project, prefix: true validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create validates :sha, presence: { unless: :importing? } diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index ac75c6501ee..eea494a22b6 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -27,6 +27,14 @@ describe Ci::Pipeline, :mailer do it { is_expected.to respond_to :git_author_email } it { is_expected.to respond_to :short_sha } + describe '#project_deployment_variables' do + it 'delegates deployment variables to project' do + expect(pipeline) + .to delegate_method(:deployment_variables) + .to(:project).with_prefix + end + end + describe '#source' do context 'when creating new pipeline' do let(:pipeline) do -- cgit v1.2.3 From ae99f74b77fc0f49a9efd5f71119e7de4e313629 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 26 Aug 2017 11:18:18 +0200 Subject: Improve kubernetes service specs readability --- spec/models/project_services/kubernetes_service_spec.rb | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/spec/models/project_services/kubernetes_service_spec.rb b/spec/models/project_services/kubernetes_service_spec.rb index b1743cd608e..537cdadd528 100644 --- a/spec/models/project_services/kubernetes_service_spec.rb +++ b/spec/models/project_services/kubernetes_service_spec.rb @@ -203,18 +203,13 @@ describe KubernetesService, :use_clean_rails_memory_store_caching do describe '#predefined_variables' do let(:kubeconfig) do - config = - YAML.load(File.read(expand_fixture_path('config/kubeconfig.yml'))) - - config.dig('users', 0, 'user')['token'] = - 'token' - + config_file = expand_fixture_path('config/kubeconfig.yml') + config = YAML.load(File.read(config_file)) + config.dig('users', 0, 'user')['token'] = 'token' + config.dig('contexts', 0, 'context')['namespace'] = namespace config.dig('clusters', 0, 'cluster')['certificate-authority-data'] = Base64.encode64('CA PEM DATA') - config.dig('contexts', 0, 'context')['namespace'] = - namespace - YAML.dump(config) end -- cgit v1.2.3 From 326dc7da3bb7e6537095277dc8ee8ae880774b62 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Sat, 26 Aug 2017 12:45:36 +0200 Subject: Check if kubernetes required before creating a job --- app/models/ci/pipeline.rb | 10 +++++++ lib/ci/gitlab_ci_yaml_processor.rb | 19 +++++++++++-- lib/gitlab/ci/config/entry/policy.rb | 11 ++++---- spec/lib/ci/gitlab_ci_yaml_processor_spec.rb | 40 ++++++++++++++++++++++++++++ spec/models/ci/pipeline_spec.rb | 40 ++++++++++++++++++++++++++++ 5 files changed, 112 insertions(+), 8 deletions(-) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 36381956195..ed18a8631c8 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -32,6 +32,7 @@ module Ci delegate :id, to: :project, prefix: true delegate :deployment_variables, to: :project, prefix: true + delegate :secret_variables_for, to: :project, prefix: true validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create validates :sha, presence: { unless: :importing? } @@ -305,6 +306,15 @@ module Ci @stage_seeds ||= config_processor.stage_seeds(self) end + def variables + project_secret_variables_for(ref: ref).map(&:to_runner_variable) + + project_deployment_variables + end + + def has_kubernetes_available? + (variables.map { |v| v.fetch(:key) } & %w[KUBECONFIG KUBE_DOMAIN]).many? + end + def has_stage_seeds? stage_seeds.any? end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 3efd9b3bdac..72a38e97648 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -44,6 +44,22 @@ module Ci end end + def pipeline_stage_builds(stage, pipeline) + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, pipeline.source) + + builds.select do |build| + job = @jobs[build.fetch(:name).to_sym] + has_kubernetes = pipeline.has_kubernetes_available? + only_kubernetes = job.dig(:only, :kubernetes) + except_kubernetes = job.dig(:except, :kubernetes) + + [!only_kubernetes & !except_kubernetes, + only_kubernetes & has_kubernetes, + except_kubernetes & !has_kubernetes].any? + end + end + def builds @jobs.map do |name, _| build_attributes(name) @@ -52,8 +68,7 @@ module Ci def stage_seeds(pipeline) seeds = @stages.uniq.map do |stage| - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) + builds = pipeline_stage_builds(stage, pipeline) Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? end diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index bcb76de65b2..a8bba3d3ea4 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -7,7 +7,7 @@ module Gitlab # class Policy < Simplifiable strategy :RefsPolicy, if: -> (config) { config.is_a?(Array) } - strategy :ExpressionsPolicy, if: -> (config) { config.is_a?(Hash) } + strategy :ComplexPolicy, if: -> (config) { config.is_a?(Hash) } class RefsPolicy < Entry::Node include Entry::Validatable @@ -21,20 +21,19 @@ module Gitlab end end - class ExpressionsPolicy < Entry::Node + class ComplexPolicy < Entry::Node include Entry::Validatable include Entry::Attributable - attributes :refs, :expressions + attributes :refs, :kubernetes validations do validates :config, presence: true - validates :config, allowed_keys: %i[refs expressions] + validates :config, allowed_keys: %i[refs kubernetes] with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true - validates :expressions, type: Array - validates :expressions, presence: true + validates :kubernetes, inclusion: { in: [true] } end end end diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index c70a4cb55fe..6d31ed875c5 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -164,6 +164,46 @@ module Ci expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' end end + + context 'when kubernetes policy is specified' do + let(:pipeline) { create(:ci_empty_pipeline) } + + let(:config) do + YAML.dump( + spinach: { stage: 'test', script: 'spinach' }, + production: { stage: 'deploy', script: 'cap', only: { + kubernetes: true } } + ) + end + + context 'when kubernetes is configured' do + let(:project) { create(:kubernetes_project) } + let(:pipeline) { create(:ci_empty_pipeline, project: project) } + + before do + create(:ci_variable, key: 'KUBE_DOMAIN', + protected: false, + project: project) + end + + it 'returns seeds for kubernetes dependent job' do + seeds = subject.stage_seeds(pipeline) + + expect(seeds.size).to eq 2 + expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' + expect(seeds.second.builds.dig(0, :name)).to eq 'production' + end + end + + context 'when kubernetes is not configured' do + it 'does not return seeds for kubernetes dependent job' do + seeds = subject.stage_seeds(pipeline) + + expect(seeds.size).to eq 1 + expect(seeds.first.builds.dig(0, :name)).to eq 'spinach' + end + end + end end describe "#builds_for_ref" do diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index eea494a22b6..b298b0b1354 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -542,6 +542,46 @@ describe Ci::Pipeline, :mailer do end end + context 'when kubernetes is configured' do + let(:project) { create(:kubernetes_project) } + + before do + create(:ci_variable, key: 'KUBE_DOMAIN', + protected: false, + project: project) + end + + describe '#variables' do + it 'returns kubernetes-related variables' do + variables = pipeline.variables.map { |v| v.fetch(:key) } + + expect(variables).to include 'KUBECONFIG', 'KUBE_DOMAIN' + end + end + + describe '#has_kubernetes_available?' do + it 'returns true' do + expect(pipeline).to have_kubernetes_available + end + end + end + + context 'when kubernetes is not configured' do + describe '#variables' do + it 'does not return kubernetes related variables' do + variables = pipeline.variables.map { |v| v.fetch(:key) } + + expect(variables).not_to include 'KUBECONFIG', 'KUBE_DOMAIN' + end + end + + describe '#has_kubernetes_available?' do + it 'returns false' do + expect(pipeline).not_to have_kubernetes_available + end + end + end + describe '#has_stage_seeds?' do context 'when pipeline has stage seeds' do subject { build(:ci_pipeline_with_one_job) } -- cgit v1.2.3 From 96a7e1628a895dcc39e6299052cb78e62d59459d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Aug 2017 13:24:03 +0200 Subject: Refactor context variables in pipeline model class --- app/models/ci/pipeline.rb | 16 ++++++++++------ spec/models/ci/pipeline_spec.rb | 16 ++++------------ 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index ed18a8631c8..248101e57d6 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -31,8 +31,6 @@ module Ci has_many :auto_canceled_jobs, class_name: 'CommitStatus', foreign_key: 'auto_canceled_by_id' delegate :id, to: :project, prefix: true - delegate :deployment_variables, to: :project, prefix: true - delegate :secret_variables_for, to: :project, prefix: true validates :source, exclusion: { in: %w(unknown), unless: :importing? }, on: :create validates :sha, presence: { unless: :importing? } @@ -306,13 +304,19 @@ module Ci @stage_seeds ||= config_processor.stage_seeds(self) end - def variables - project_secret_variables_for(ref: ref).map(&:to_runner_variable) + - project_deployment_variables + def context_variables + @context_variables ||= project.secret_variables_for(ref: ref).to_a + .map(&:to_runner_variable) + project.deployment_variables.to_a end def has_kubernetes_available? - (variables.map { |v| v.fetch(:key) } & %w[KUBECONFIG KUBE_DOMAIN]).many? + kubernetes_variables = context_variables.select do |variable| + variable.fetch(:key).in?(%w[KUBECONFIG KUBE_DOMAIN]) + end + + return false if kubernetes_variables.empty? + + kubernetes_variables.map { |var| var.fetch(:value).present? }.all? end def has_stage_seeds? diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index b298b0b1354..11838b1ee07 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -27,14 +27,6 @@ describe Ci::Pipeline, :mailer do it { is_expected.to respond_to :git_author_email } it { is_expected.to respond_to :short_sha } - describe '#project_deployment_variables' do - it 'delegates deployment variables to project' do - expect(pipeline) - .to delegate_method(:deployment_variables) - .to(:project).with_prefix - end - end - describe '#source' do context 'when creating new pipeline' do let(:pipeline) do @@ -551,9 +543,9 @@ describe Ci::Pipeline, :mailer do project: project) end - describe '#variables' do + describe '#context_variables' do it 'returns kubernetes-related variables' do - variables = pipeline.variables.map { |v| v.fetch(:key) } + variables = pipeline.context_variables.map { |v| v.fetch(:key) } expect(variables).to include 'KUBECONFIG', 'KUBE_DOMAIN' end @@ -567,9 +559,9 @@ describe Ci::Pipeline, :mailer do end context 'when kubernetes is not configured' do - describe '#variables' do + describe '#context_variables' do it 'does not return kubernetes related variables' do - variables = pipeline.variables.map { |v| v.fetch(:key) } + variables = pipeline.context_variables.map { |v| v.fetch(:key) } expect(variables).not_to include 'KUBECONFIG', 'KUBE_DOMAIN' end -- cgit v1.2.3 From 9eabd7230da59708981a9ab5f176beaf97aa6923 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Aug 2017 13:48:23 +0200 Subject: Remove unused method from CI/CD YAML processor --- lib/ci/gitlab_ci_yaml_processor.rb | 6 ------ spec/lib/ci/gitlab_ci_yaml_processor_spec.rb | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 72a38e97648..e767ffd2211 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -32,12 +32,6 @@ module Ci end end - def builds_for_ref(ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).map do |name, _| - build_attributes(name) - end - end - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| build_attributes(name) diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index 6d31ed875c5..dbe30807595 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -206,7 +206,7 @@ module Ci end end - describe "#builds_for_ref" do + describe "#builds_for_stage_and_ref" do let(:type) { 'test' } it "returns builds if no branch specified" do -- cgit v1.2.3 From 2339306df6e642dfb2c8423e69a1f67a064a2119 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Aug 2017 13:48:42 +0200 Subject: Improve matching kubernetes job in CI/CD processor --- lib/ci/gitlab_ci_yaml_processor.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index e767ffd2211..8dff879018f 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -48,9 +48,9 @@ module Ci only_kubernetes = job.dig(:only, :kubernetes) except_kubernetes = job.dig(:except, :kubernetes) - [!only_kubernetes & !except_kubernetes, - only_kubernetes & has_kubernetes, - except_kubernetes & !has_kubernetes].any? + [!only_kubernetes && !except_kubernetes, + only_kubernetes && has_kubernetes, + except_kubernetes && !has_kubernetes].any? end end -- cgit v1.2.3 From 5623cb9a6eb0075306b1cecfd336c2b5ce997f19 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Aug 2017 13:50:52 +0200 Subject: Make some methods in CI/CD YAML processor private --- lib/ci/gitlab_ci_yaml_processor.rb | 56 ++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 8dff879018f..5676618f7fd 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -20,17 +20,6 @@ module Ci raise ValidationError, e.message end - def jobs_for_ref(ref, tag = false, source = nil) - @jobs.select do |_, job| - process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) - end - end - - def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).select do |_, job| - job[:stage] == stage - end - end def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| @@ -38,22 +27,6 @@ module Ci end end - def pipeline_stage_builds(stage, pipeline) - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) - - builds.select do |build| - job = @jobs[build.fetch(:name).to_sym] - has_kubernetes = pipeline.has_kubernetes_available? - only_kubernetes = job.dig(:only, :kubernetes) - except_kubernetes = job.dig(:except, :kubernetes) - - [!only_kubernetes && !except_kubernetes, - only_kubernetes && has_kubernetes, - except_kubernetes && !has_kubernetes].any? - end - end - def builds @jobs.map do |name, _| build_attributes(name) @@ -110,6 +83,35 @@ module Ci private + def pipeline_stage_builds(stage, pipeline) + builds = builds_for_stage_and_ref( + stage, pipeline.ref, pipeline.tag?, pipeline.source) + + builds.select do |build| + job = @jobs[build.fetch(:name).to_sym] + has_kubernetes = pipeline.has_kubernetes_available? + only_kubernetes = job.dig(:only, :kubernetes) + except_kubernetes = job.dig(:except, :kubernetes) + + [!only_kubernetes && !except_kubernetes, + only_kubernetes && has_kubernetes, + except_kubernetes && !has_kubernetes].any? + end + end + + + def jobs_for_ref(ref, tag = false, source = nil) + @jobs.select do |_, job| + process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) + end + end + + def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) + jobs_for_ref(ref, tag, source).select do |_, job| + job[:stage] == stage + end + end + def initial_parsing ## # Global config -- cgit v1.2.3 From ef030709ebffdecdce67f2693d8c54669edfb7a7 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Aug 2017 14:56:25 +0200 Subject: Change kubernetes job policy allowed values It is now possible to use `kubernetes: configured`. --- lib/ci/gitlab_ci_yaml_processor.rb | 5 ++- lib/gitlab/ci/config/entry/policy.rb | 2 +- lib/gitlab/ci/config/entry/validators.rb | 8 +++++ spec/lib/ci/gitlab_ci_yaml_processor_spec.rb | 2 +- spec/lib/gitlab/ci/config/entry/policy_spec.rb | 44 ++++++++++++++++++++++++++ 5 files changed, 56 insertions(+), 5 deletions(-) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 5676618f7fd..434b8948f41 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -94,12 +94,11 @@ module Ci except_kubernetes = job.dig(:except, :kubernetes) [!only_kubernetes && !except_kubernetes, - only_kubernetes && has_kubernetes, - except_kubernetes && !has_kubernetes].any? + only_kubernetes && has_kubernetes, + except_kubernetes && !has_kubernetes].any? end end - def jobs_for_ref(ref, tag = false, source = nil) @jobs.select do |_, job| process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index a8bba3d3ea4..922d568f0ab 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -33,7 +33,7 @@ module Gitlab with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true - validates :kubernetes, inclusion: { in: [true] } + validates :kubernetes, allowed_values: %w[configured] end end end diff --git a/lib/gitlab/ci/config/entry/validators.rb b/lib/gitlab/ci/config/entry/validators.rb index b2ca3c881e4..0159179f0a9 100644 --- a/lib/gitlab/ci/config/entry/validators.rb +++ b/lib/gitlab/ci/config/entry/validators.rb @@ -14,6 +14,14 @@ module Gitlab end end + class AllowedValuesValidator < ActiveModel::EachValidator + def validate_each(record, attribute, value) + unless options[:in].include?(value.to_s) + record.errors.add(attribute, "unknown value: #{value}") + end + end + end + class ArrayOfStringsValidator < ActiveModel::EachValidator include LegacyValidationHelpers diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index dbe30807595..e8908ebec8d 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -172,7 +172,7 @@ module Ci YAML.dump( spinach: { stage: 'test', script: 'spinach' }, production: { stage: 'deploy', script: 'cap', only: { - kubernetes: true } } + kubernetes: 'configured' } } ) end diff --git a/spec/lib/gitlab/ci/config/entry/policy_spec.rb b/spec/lib/gitlab/ci/config/entry/policy_spec.rb index ab69a2e8bcc..a5c2e3f1989 100644 --- a/spec/lib/gitlab/ci/config/entry/policy_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/policy_spec.rb @@ -56,6 +56,50 @@ describe Gitlab::Ci::Config::Entry::Policy do end end + context 'when using complex policy' do + context 'when specifiying refs policy' do + let(:config) { { refs: ['master'] } } + + it 'is a correct configuraton' do + expect(entry).to be_valid + expect(entry.value).to eq(refs: %w[master]) + end + end + + context 'when specifying kubernetes policy' do + let(:config) { { kubernetes: 'configured' } } + + it 'is a correct configuraton' do + expect(entry).to be_valid + expect(entry.value).to eq(kubernetes: 'configured') + end + end + + context 'when specifying invalid kubernetes policy' do + let(:config) { { kubernetes: 'active' } } + + it 'reports an error about invalid policy' do + expect(entry.errors).to include /unknown value: active/ + end + end + + context 'when specifying unknown policy' do + let(:config) { { refs: ['master'], invalid: :something } } + + it 'returns error about invalid key' do + expect(entry.errors).to include /unknown keys: invalid/ + end + end + + context 'when policy is empty' do + let(:config) { {} } + + it 'is not a valid configuration' do + expect(entry.errors).to include /can't be blank/ + end + end + end + context 'when policy strategy does not match' do let(:config) { 'string strategy' } -- cgit v1.2.3 From dc4d938676414579ad8cfca9689439668c90b411 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 31 Aug 2017 15:30:48 +0200 Subject: Fix only/except policy CI/CD linter --- app/views/ci/lints/_create.html.haml | 4 ++-- spec/views/ci/lints/show.html.haml_spec.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/views/ci/lints/_create.html.haml b/app/views/ci/lints/_create.html.haml index c91602fcff7..30bf1384b22 100644 --- a/app/views/ci/lints/_create.html.haml +++ b/app/views/ci/lints/_create.html.haml @@ -22,10 +22,10 @@ %b Tag list: = build[:tag_list].to_a.join(", ") %br - %b Refs only: + %b Only policy: = @jobs[build[:name].to_sym][:only].to_a.join(", ") %br - %b Refs except: + %b Except policy: = @jobs[build[:name].to_sym][:except].to_a.join(", ") %br %b Environment: diff --git a/spec/views/ci/lints/show.html.haml_spec.rb b/spec/views/ci/lints/show.html.haml_spec.rb index 3390ae247ff..f2c19c7642a 100644 --- a/spec/views/ci/lints/show.html.haml_spec.rb +++ b/spec/views/ci/lints/show.html.haml_spec.rb @@ -73,8 +73,8 @@ describe 'ci/lints/show' do render expect(rendered).to have_content('Tag list: dotnet') - expect(rendered).to have_content('Refs only: test@dude/repo') - expect(rendered).to have_content('Refs except: deploy') + expect(rendered).to have_content('Only policy: refs, test@dude/repo') + expect(rendered).to have_content('Except policy: refs, deploy') expect(rendered).to have_content('Environment: testing') expect(rendered).to have_content('When: on_success') end -- cgit v1.2.3 From 73d0cafbe4acf5a14ec196c3cfb38be4e84b67d9 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 1 Sep 2017 12:58:22 +0200 Subject: Check if service active when using CI/CD kubernetes policy --- app/models/ci/pipeline.rb | 15 ++------------- lib/ci/gitlab_ci_yaml_processor.rb | 2 +- spec/models/ci/pipeline_spec.rb | 36 ++++++------------------------------ 3 files changed, 9 insertions(+), 44 deletions(-) diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 65e542106e3..46e5c344fdc 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -304,19 +304,8 @@ module Ci @stage_seeds ||= config_processor.stage_seeds(self) end - def context_variables - @context_variables ||= project.secret_variables_for(ref: ref).to_a - .map(&:to_runner_variable) + project.deployment_variables.to_a - end - - def has_kubernetes_available? - kubernetes_variables = context_variables.select do |variable| - variable.fetch(:key).in?(%w[KUBECONFIG KUBE_DOMAIN]) - end - - return false if kubernetes_variables.empty? - - kubernetes_variables.map { |var| var.fetch(:value).present? }.all? + def has_kubernetes_active? + project.kubernetes_service&.active? end def has_stage_seeds? diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 434b8948f41..d3551907969 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -89,7 +89,7 @@ module Ci builds.select do |build| job = @jobs[build.fetch(:name).to_sym] - has_kubernetes = pipeline.has_kubernetes_available? + has_kubernetes = pipeline.has_kubernetes_active? only_kubernetes = job.dig(:only, :kubernetes) except_kubernetes = job.dig(:except, :kubernetes) diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index 7898bdbe2ff..cf4896ff188 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -546,42 +546,18 @@ describe Ci::Pipeline, :mailer do end end - context 'when kubernetes is configured' do - let(:project) { create(:kubernetes_project) } + describe '#has_kubernetes_active?' do + context 'when kubernetes is configured' do + let(:project) { create(:kubernetes_project) } - before do - create(:ci_variable, key: 'KUBE_DOMAIN', - protected: false, - project: project) - end - - describe '#context_variables' do - it 'returns kubernetes-related variables' do - variables = pipeline.context_variables.map { |v| v.fetch(:key) } - - expect(variables).to include 'KUBECONFIG', 'KUBE_DOMAIN' - end - end - - describe '#has_kubernetes_available?' do it 'returns true' do - expect(pipeline).to have_kubernetes_available - end - end - end - - context 'when kubernetes is not configured' do - describe '#context_variables' do - it 'does not return kubernetes related variables' do - variables = pipeline.context_variables.map { |v| v.fetch(:key) } - - expect(variables).not_to include 'KUBECONFIG', 'KUBE_DOMAIN' + expect(pipeline).to have_kubernetes_active end end - describe '#has_kubernetes_available?' do + context 'when kubernetes is not configured' do it 'returns false' do - expect(pipeline).not_to have_kubernetes_available + expect(pipeline).not_to have_kubernetes_active end end end -- cgit v1.2.3 From fc4fb6e4d212d5cf406a08c96ba8c423477a5408 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 1 Sep 2017 13:03:43 +0200 Subject: Change CI/CD kubernetes policy keyword to `active` --- lib/gitlab/ci/config/entry/policy.rb | 2 +- spec/lib/ci/gitlab_ci_yaml_processor_spec.rb | 12 +++--------- spec/lib/gitlab/ci/config/entry/policy_spec.rb | 8 ++++---- spec/models/ci/pipeline_spec.rb | 4 ++-- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/lib/gitlab/ci/config/entry/policy.rb b/lib/gitlab/ci/config/entry/policy.rb index 922d568f0ab..0027e9ec8c5 100644 --- a/lib/gitlab/ci/config/entry/policy.rb +++ b/lib/gitlab/ci/config/entry/policy.rb @@ -33,7 +33,7 @@ module Gitlab with_options allow_nil: true do validates :refs, array_of_strings_or_regexps: true - validates :kubernetes, allowed_values: %w[configured] + validates :kubernetes, allowed_values: %w[active] end end end diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index e8908ebec8d..c25800206b8 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -172,20 +172,14 @@ module Ci YAML.dump( spinach: { stage: 'test', script: 'spinach' }, production: { stage: 'deploy', script: 'cap', only: { - kubernetes: 'configured' } } + kubernetes: 'active' } } ) end - context 'when kubernetes is configured' do + context 'when kubernetes is active' do let(:project) { create(:kubernetes_project) } let(:pipeline) { create(:ci_empty_pipeline, project: project) } - before do - create(:ci_variable, key: 'KUBE_DOMAIN', - protected: false, - project: project) - end - it 'returns seeds for kubernetes dependent job' do seeds = subject.stage_seeds(pipeline) @@ -195,7 +189,7 @@ module Ci end end - context 'when kubernetes is not configured' do + context 'when kubernetes is not active' do it 'does not return seeds for kubernetes dependent job' do seeds = subject.stage_seeds(pipeline) diff --git a/spec/lib/gitlab/ci/config/entry/policy_spec.rb b/spec/lib/gitlab/ci/config/entry/policy_spec.rb index a5c2e3f1989..5e83abf645b 100644 --- a/spec/lib/gitlab/ci/config/entry/policy_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/policy_spec.rb @@ -67,19 +67,19 @@ describe Gitlab::Ci::Config::Entry::Policy do end context 'when specifying kubernetes policy' do - let(:config) { { kubernetes: 'configured' } } + let(:config) { { kubernetes: 'active' } } it 'is a correct configuraton' do expect(entry).to be_valid - expect(entry.value).to eq(kubernetes: 'configured') + expect(entry.value).to eq(kubernetes: 'active') end end context 'when specifying invalid kubernetes policy' do - let(:config) { { kubernetes: 'active' } } + let(:config) { { kubernetes: 'something' } } it 'reports an error about invalid policy' do - expect(entry.errors).to include /unknown value: active/ + expect(entry.errors).to include /unknown value: something/ end end diff --git a/spec/models/ci/pipeline_spec.rb b/spec/models/ci/pipeline_spec.rb index cf4896ff188..84656ffe0b9 100644 --- a/spec/models/ci/pipeline_spec.rb +++ b/spec/models/ci/pipeline_spec.rb @@ -547,7 +547,7 @@ describe Ci::Pipeline, :mailer do end describe '#has_kubernetes_active?' do - context 'when kubernetes is configured' do + context 'when kubernetes is active' do let(:project) { create(:kubernetes_project) } it 'returns true' do @@ -555,7 +555,7 @@ describe Ci::Pipeline, :mailer do end end - context 'when kubernetes is not configured' do + context 'when kubernetes is not active' do it 'returns false' do expect(pipeline).not_to have_kubernetes_active end -- cgit v1.2.3 From 6f6583923f757b830520846f3a5ae8c6983dc40b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 1 Sep 2017 13:10:07 +0200 Subject: Add changelog entry for kubernetes: active job policy --- changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml diff --git a/changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml b/changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml new file mode 100644 index 00000000000..00c38a0c671 --- /dev/null +++ b/changelogs/unreleased/feature-gb-kubernetes-only-pipeline-jobs.yml @@ -0,0 +1,5 @@ +--- +title: Add CI/CD active kubernetes job policy +merge_request: 13849 +author: +type: added -- cgit v1.2.3 From 84827c1387a66970cbbbec28ae7ea56e9e4fa01e Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 1 Sep 2017 13:42:10 +0200 Subject: Add some basic docs about kubernetes: active feature --- doc/ci/yaml/README.md | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index abf4ec7dbf8..eaf2c557a1e 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -427,16 +427,16 @@ a "key: value" pair. Be careful when using special characters: are executed in `parallel`. For more info about the use of `stage` please check [stages](#stages). -### only and except +### only and except (simplified) -`only` and `except` are two parameters that set a refs policy to limit when -jobs are built: +`only` and `except` are two parameters that set a job policy to limit when +jobs are created: 1. `only` defines the names of branches and tags for which the job will run. 2. `except` defines the names of branches and tags for which the job will **not** run. -There are a few rules that apply to the usage of refs policy: +There are a few rules that apply to the usage of job policy: * `only` and `except` are inclusive. If both `only` and `except` are defined in a job specification, the ref is filtered by `only` and `except`. @@ -497,6 +497,33 @@ job: The above example will run `job` for all branches on `gitlab-org/gitlab-ce`, except master. +### only and except (complex) + +> Introduced in GitLab 10.0 + +Since GitLab 10.0 it is possible to define a complex only/except job policy +configuration. + +GitLab now supports both, simple and complex strategies, so it is possible to +use array and hash scheme configuration. + +Two keys are now available: `refs` and `kubernetes`. Refs strategy equals to +simplified only/except configuration, whereas kubernetes strategy accepts only +`active` keyword. + +See the example below. Job is going to be created only when pipeline has been +scheduled or runs for a `master` branch, and only if kubernetes service is +active in the project. + +```yaml +job: + only: + refs: + - master + - schedules + kubernetes: active +``` + ### Job variables It is possible to define job variables using a `variables` keyword on a job -- cgit v1.2.3 From f911dafe6621c9735a11c6289bc1ef62a302cd7a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 1 Sep 2017 13:44:15 +0200 Subject: Add disclaimer about alpha kubernetes: active feature --- doc/ci/yaml/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index eaf2c557a1e..b281ff386bd 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -500,6 +500,7 @@ except master. ### only and except (complex) > Introduced in GitLab 10.0 +> This an _alpha_ feature, and it it subject to change at any time without prior notice! Since GitLab 10.0 it is possible to define a complex only/except job policy configuration. -- cgit v1.2.3 From ab8e6c802803dcbfb2624742d79439718deb1956 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 1 Sep 2017 13:47:58 +0200 Subject: Copy-edit kubernetes: active feature docs --- doc/ci/yaml/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index b281ff386bd..bcf5b5d0b5f 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -500,13 +500,14 @@ except master. ### only and except (complex) > Introduced in GitLab 10.0 -> This an _alpha_ feature, and it it subject to change at any time without prior notice! +> This an _alpha_ feature, and it it subject to change at any time without + prior notice! -Since GitLab 10.0 it is possible to define a complex only/except job policy -configuration. +Since GitLab 10.0 it is possible to define a more elaborate only/except job +policy configuration. GitLab now supports both, simple and complex strategies, so it is possible to -use array and hash scheme configuration. +use an array and a hash configuration scheme. Two keys are now available: `refs` and `kubernetes`. Refs strategy equals to simplified only/except configuration, whereas kubernetes strategy accepts only -- cgit v1.2.3 From 2c5a4cf571a1ee3c7f09c4fc853d609f7e1c826d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 1 Sep 2017 11:51:05 +0000 Subject: Fix only/except docs disclaimer's markdown --- doc/ci/yaml/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index bcf5b5d0b5f..cacfd2ed254 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -500,6 +500,7 @@ except master. ### only and except (complex) > Introduced in GitLab 10.0 + > This an _alpha_ feature, and it it subject to change at any time without prior notice! -- cgit v1.2.3 From a6ead9d17966dd0fca7c5810947eebf818d524b3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 1 Sep 2017 14:00:41 +0200 Subject: Remove an empty line from YAML processor class --- lib/ci/gitlab_ci_yaml_processor.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index d3551907969..62b44389b15 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -20,7 +20,6 @@ module Ci raise ValidationError, e.message end - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| build_attributes(name) -- cgit v1.2.3 From 339a1ad011f87fa969fd5ba644c504518924771a Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 4 Sep 2017 10:21:36 +0200 Subject: Fix rubocop offense in CI/CD YAML processor specs --- spec/lib/ci/gitlab_ci_yaml_processor_spec.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb index c25800206b8..1efd3113a43 100644 --- a/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb +++ b/spec/lib/ci/gitlab_ci_yaml_processor_spec.rb @@ -171,8 +171,11 @@ module Ci let(:config) do YAML.dump( spinach: { stage: 'test', script: 'spinach' }, - production: { stage: 'deploy', script: 'cap', only: { - kubernetes: 'active' } } + production: { + stage: 'deploy', + script: 'cap', + only: { kubernetes: 'active' } + } ) end -- cgit v1.2.3