From 59f87e7317137c302f71eb6746262c1f4fee84d9 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 14 Sep 2017 10:52:25 +0200 Subject: Implement CI/CD kubernetes policy specification --- lib/gitlab/ci/build/policy/kubernetes.rb | 19 +++++++++++++++++++ lib/gitlab/ci/build/policy/specification.rb | 28 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 lib/gitlab/ci/build/policy/kubernetes.rb create mode 100644 lib/gitlab/ci/build/policy/specification.rb (limited to 'lib/gitlab') diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb new file mode 100644 index 00000000000..bf3a95e08f0 --- /dev/null +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Build + module Policy + class Kubernetes < Policy::Specification + def initialize(spec) + unless spec.to_sym == :active + raise UnknownPolicyError + end + end + + def satisfied_by?(pipeline, **_) + pipeline.has_kubernetes_active? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb new file mode 100644 index 00000000000..0bb06ee8381 --- /dev/null +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -0,0 +1,28 @@ +module Gitlab + module Ci + module Build + module Policy + ## + # Abstract class that defines an intereface of job policy + # specification. + # + # Used for job's only/except policy configuration. + # + class Specification + UnknownPolicyError = Class.new(StandardError) + + def initialize(spec) + @spec = spec + end + + def satisfied_by?(pipeline, **metadata) + raise NotImplementedError + end + + def self.fabricate_all(*specs) + end + end + end + end + end +end -- cgit v1.2.3 From dd784b1518501032e01e521adbf6ecf98d54ba55 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 14 Sep 2017 12:10:09 +0200 Subject: Use CI/CD job policy specification in YAML processor --- lib/gitlab/ci/build/policy.rb | 19 +++++++++++++++++++ lib/gitlab/ci/build/policy/specification.rb | 3 --- lib/gitlab/ci/yaml_processor.rb | 13 +++++++------ 3 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 lib/gitlab/ci/build/policy.rb (limited to 'lib/gitlab') diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb new file mode 100644 index 00000000000..6ab2f86c2e9 --- /dev/null +++ b/lib/gitlab/ci/build/policy.rb @@ -0,0 +1,19 @@ +module Gitlab + module Ci + module Build + module Policy + def self.fabricate(specs) + specifications = specs.to_h.map do |spec, value| + begin + self.const_get(spec.to_s.camelize).new(value) + rescue NameError + next + end + end + + specifications.compact + end + end + end + end +end diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb index 0bb06ee8381..9ca3582b9b0 100644 --- a/lib/gitlab/ci/build/policy/specification.rb +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -18,9 +18,6 @@ module Gitlab def satisfied_by?(pipeline, **metadata) raise NotImplementedError end - - def self.fabricate_all(*specs) - end end end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 7582964b24e..44e616c01e5 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -89,13 +89,14 @@ module Gitlab builds.select do |build| job = @jobs[build.fetch(:name).to_sym] - has_kubernetes = pipeline.has_kubernetes_active? - only_kubernetes = job.dig(:only, :kubernetes) - except_kubernetes = job.dig(:except, :kubernetes) - [!only_kubernetes && !except_kubernetes, - only_kubernetes && has_kubernetes, - except_kubernetes && !has_kubernetes].any? + only_specs = Gitlab::Ci::Build::Policy + .fabricate(job.fetch(:only, {})) + except_specs = Gitlab::Ci::Build::Policy + .fabricate(job.fetch(:except, {})) + + only_specs.all? { |spec| spec.satisfied_by?(pipeline) } && + except_specs.none? { |spec| spec.satisfied_by?(pipeline) } end end -- cgit v1.2.3 From 241197c29aa8fe3cae9e5ded9947cfb2bee3574c Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 14 Sep 2017 12:59:41 +0200 Subject: Extract job refs policy specs into separate class --- lib/gitlab/ci/build/policy/refs.rb | 50 +++++++++++++++++++++++++ lib/gitlab/ci/yaml_processor.rb | 75 +++++--------------------------------- 2 files changed, 59 insertions(+), 66 deletions(-) create mode 100644 lib/gitlab/ci/build/policy/refs.rb (limited to 'lib/gitlab') diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb new file mode 100644 index 00000000000..4f2bbd9eaac --- /dev/null +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -0,0 +1,50 @@ +module Gitlab + module Ci + module Build + module Policy + class Refs < Policy::Specification + def initialize(refs) + @patterns = Array(refs) + end + + def satisfied_by?(pipeline, path:) + @patterns.any? do |pattern| + pattern, ref_path = pattern.split('@', 2) + + matches_path?(ref_path, path) && + matches_pattern?(pattern, pipeline) + end + end + + private + + def matches_path?(ref_path, expected_path) + return true unless ref_path + + expected_path == ref_path + end + + def matches_pattern?(pattern, pipeline) + return true if pipeline.tag? && pattern == 'tags' + return true if !pipeline.tag? && pattern == 'branches' + return true if source_to_pattern(pipeline.source) == pattern + + if pattern.first == "/" && pattern.last == "/" + Regexp.new(pattern[1...-1]) =~ pipeline.ref + else + pattern == pipeline.ref + end + end + + def source_to_pattern(source) + if %w[api external web].include?(source) + source + else + source&.pluralize + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 44e616c01e5..d688683474c 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -21,10 +21,11 @@ module Gitlab raise ValidationError, e.message end + + # REFACTORING STUB, remove this method, used only in tests. + # def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_stage_and_ref(stage, ref, tag, source).map do |name, _| - build_attributes(name) - end + pipeline_stage_builds(stage, ::Ci::Pipeline.new(ref: ref, source: source, tag: tag)) end def builds @@ -84,32 +85,19 @@ module Gitlab private def pipeline_stage_builds(stage, pipeline) - builds = builds_for_stage_and_ref( - stage, pipeline.ref, pipeline.tag?, pipeline.source) - - builds.select do |build| - job = @jobs[build.fetch(:name).to_sym] + stage_jobs = @jobs.select do |_, job| + next unless job[:stage] == stage only_specs = Gitlab::Ci::Build::Policy .fabricate(job.fetch(:only, {})) except_specs = Gitlab::Ci::Build::Policy .fabricate(job.fetch(:except, {})) - only_specs.all? { |spec| spec.satisfied_by?(pipeline) } && - except_specs.none? { |spec| spec.satisfied_by?(pipeline) } - end - end - - def jobs_for_ref(ref, tag = false, source = nil) - @jobs.select do |_, job| - process?(job.dig(:only, :refs), job.dig(:except, :refs), ref, tag, source) + only_specs.all? { |spec| spec.satisfied_by?(pipeline, path: @path) } && + except_specs.none? { |spec| spec.satisfied_by?(pipeline, path: @path) } end - end - def jobs_for_stage_and_ref(stage, ref, tag = false, source = nil) - jobs_for_ref(ref, tag, source).select do |_, job| - job[:stage] == stage - end + stage_jobs.map { |_, job| build_attributes(job[:name]) } end def initial_parsing @@ -204,51 +192,6 @@ module Gitlab raise ValidationError, "#{name} job: on_stop job #{on_stop} needs to have action stop defined" end end - - def process?(only_params, except_params, ref, tag, source) - if only_params.present? - return false unless matching?(only_params, ref, tag, source) - end - - if except_params.present? - return false if matching?(except_params, ref, tag, source) - end - - true - end - - def matching?(patterns, ref, tag, source) - patterns.any? do |pattern| - pattern, path = pattern.split('@', 2) - matches_path?(path) && matches_pattern?(pattern, ref, tag, source) - end - end - - def matches_path?(path) - return true unless path - - path == self.path - end - - def matches_pattern?(pattern, ref, tag, source) - return true if tag && pattern == 'tags' - return true if !tag && pattern == 'branches' - return true if source_to_pattern(source) == pattern - - if pattern.first == "/" && pattern.last == "/" - Regexp.new(pattern[1...-1]) =~ ref - else - pattern == ref - end - end - - def source_to_pattern(source) - if %w[api external web].include?(source) - source - else - source&.pluralize - end - end end end end -- cgit v1.2.3 From 10a486b366a834b24c6eef432ec16af4237ec06d Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 14 Sep 2017 13:53:02 +0200 Subject: Remove YAML processor refactoring stubs and fix specs --- lib/gitlab/ci/build/policy/refs.rb | 2 +- lib/gitlab/ci/yaml_processor.rb | 56 +++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 32 deletions(-) (limited to 'lib/gitlab') diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 4f2bbd9eaac..9e1b639f12d 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -26,7 +26,7 @@ module Gitlab def matches_pattern?(pattern, pipeline) return true if pipeline.tag? && pattern == 'tags' - return true if !pipeline.tag? && pattern == 'branches' + return true if pipeline.branch? && pattern == 'branches' return true if source_to_pattern(pipeline.source) == pattern if pattern.first == "/" && pattern.last == "/" diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index d688683474c..19951f1842e 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -22,28 +22,12 @@ module Gitlab end - # REFACTORING STUB, remove this method, used only in tests. - # - def builds_for_stage_and_ref(stage, ref, tag = false, source = nil) - pipeline_stage_builds(stage, ::Ci::Pipeline.new(ref: ref, source: source, tag: tag)) - end - def builds @jobs.map do |name, _| build_attributes(name) end end - def stage_seeds(pipeline) - seeds = @stages.uniq.map do |stage| - builds = pipeline_stage_builds(stage, pipeline) - - Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? - end - - seeds.compact - end - def build_attributes(name) job = @jobs[name.to_sym] || {} @@ -71,21 +55,8 @@ module Gitlab }.compact } end - def self.validation_message(content) - return 'Please provide content of .gitlab-ci.yml' if content.blank? - - begin - Gitlab::Ci::YamlProcessor.new(content) - nil - rescue ValidationError, Psych::SyntaxError => e - e.message - end - end - - private - def pipeline_stage_builds(stage, pipeline) - stage_jobs = @jobs.select do |_, job| + selected_jobs = @jobs.select do |_, job| next unless job[:stage] == stage only_specs = Gitlab::Ci::Build::Policy @@ -97,9 +68,32 @@ module Gitlab except_specs.none? { |spec| spec.satisfied_by?(pipeline, path: @path) } end - stage_jobs.map { |_, job| build_attributes(job[:name]) } + selected_jobs.map { |_, job| build_attributes(job[:name]) } + end + + def stage_seeds(pipeline) + seeds = @stages.uniq.map do |stage| + builds = pipeline_stage_builds(stage, pipeline) + + Gitlab::Ci::Stage::Seed.new(pipeline, stage, builds) if builds.any? + end + + seeds.compact + end + + def self.validation_message(content) + return 'Please provide content of .gitlab-ci.yml' if content.blank? + + begin + Gitlab::Ci::YamlProcessor.new(content) + nil + rescue ValidationError, Psych::SyntaxError => e + e.message + end end + private + def initial_parsing ## # Global config -- cgit v1.2.3 From a7a7e2d8f7779714b8d1ebdcb62d8757eeaba2bb Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 15 Sep 2017 12:12:27 +0200 Subject: Fix Rubocop offenses in YAML processor classes --- lib/gitlab/ci/yaml_processor.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib/gitlab') diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 19951f1842e..3a7a2761db8 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -21,7 +21,6 @@ module Gitlab raise ValidationError, e.message end - def builds @jobs.map do |name, _| build_attributes(name) -- cgit v1.2.3 From a04cbd5bb52dbaf6d0b05eca5215400f30051974 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Fri, 15 Sep 2017 13:29:50 +0200 Subject: Add specs for CI/CD job policy refs specification --- lib/gitlab/ci/build/policy/refs.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/gitlab') diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 9e1b639f12d..641c2e510be 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -7,7 +7,7 @@ module Gitlab @patterns = Array(refs) end - def satisfied_by?(pipeline, path:) + def satisfied_by?(pipeline, path: nil) @patterns.any? do |pattern| pattern, ref_path = pattern.split('@', 2) -- cgit v1.2.3 From 6681ea9cd8d7001a352c95237d625057b6147f0b Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 18 Sep 2017 14:00:59 +0200 Subject: Simplify pipeline source policy pattern matching --- lib/gitlab/ci/build/policy/refs.rb | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) (limited to 'lib/gitlab') diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 641c2e510be..76a57f041e3 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -27,7 +27,8 @@ module Gitlab def matches_pattern?(pattern, pipeline) return true if pipeline.tag? && pattern == 'tags' return true if pipeline.branch? && pattern == 'branches' - return true if source_to_pattern(pipeline.source) == pattern + return true if pipeline.source == pattern + return true if pipeline.source&.pluralize == pattern if pattern.first == "/" && pattern.last == "/" Regexp.new(pattern[1...-1]) =~ pipeline.ref @@ -35,14 +36,6 @@ module Gitlab pattern == pipeline.ref end end - - def source_to_pattern(source) - if %w[api external web].include?(source) - source - else - source&.pluralize - end - end end end end -- cgit v1.2.3 From d79ad28fcb44c35d77de26d428ae61bc6d71e8ec Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 18 Sep 2017 14:29:43 +0200 Subject: Do not pass project path from YAML processor Use project full path that can be received from a pipeline object --- lib/gitlab/ci/build/policy/kubernetes.rb | 2 +- lib/gitlab/ci/build/policy/refs.rb | 12 ++++++------ lib/gitlab/ci/build/policy/specification.rb | 4 ++-- lib/gitlab/ci/yaml_processor.rb | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) (limited to 'lib/gitlab') diff --git a/lib/gitlab/ci/build/policy/kubernetes.rb b/lib/gitlab/ci/build/policy/kubernetes.rb index bf3a95e08f0..b20d374288f 100644 --- a/lib/gitlab/ci/build/policy/kubernetes.rb +++ b/lib/gitlab/ci/build/policy/kubernetes.rb @@ -9,7 +9,7 @@ module Gitlab end end - def satisfied_by?(pipeline, **_) + def satisfied_by?(pipeline) pipeline.has_kubernetes_active? end end diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb index 76a57f041e3..eadc0948d2f 100644 --- a/lib/gitlab/ci/build/policy/refs.rb +++ b/lib/gitlab/ci/build/policy/refs.rb @@ -7,21 +7,21 @@ module Gitlab @patterns = Array(refs) end - def satisfied_by?(pipeline, path: nil) + def satisfied_by?(pipeline) @patterns.any? do |pattern| - pattern, ref_path = pattern.split('@', 2) + pattern, path = pattern.split('@', 2) - matches_path?(ref_path, path) && + matches_path?(path, pipeline) && matches_pattern?(pattern, pipeline) end end private - def matches_path?(ref_path, expected_path) - return true unless ref_path + def matches_path?(path, pipeline) + return true unless path - expected_path == ref_path + pipeline.project_full_path == path end def matches_pattern?(pattern, pipeline) diff --git a/lib/gitlab/ci/build/policy/specification.rb b/lib/gitlab/ci/build/policy/specification.rb index 9ca3582b9b0..c317291f29d 100644 --- a/lib/gitlab/ci/build/policy/specification.rb +++ b/lib/gitlab/ci/build/policy/specification.rb @@ -3,7 +3,7 @@ module Gitlab module Build module Policy ## - # Abstract class that defines an intereface of job policy + # Abstract class that defines an interface of job policy # specification. # # Used for job's only/except policy configuration. @@ -15,7 +15,7 @@ module Gitlab @spec = spec end - def satisfied_by?(pipeline, **metadata) + def satisfied_by?(pipeline) raise NotImplementedError end end diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 3a7a2761db8..da39a28df4c 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -63,8 +63,8 @@ module Gitlab except_specs = Gitlab::Ci::Build::Policy .fabricate(job.fetch(:except, {})) - only_specs.all? { |spec| spec.satisfied_by?(pipeline, path: @path) } && - except_specs.none? { |spec| spec.satisfied_by?(pipeline, path: @path) } + only_specs.all? { |spec| spec.satisfied_by?(pipeline) } && + except_specs.none? { |spec| spec.satisfied_by?(pipeline) } end selected_jobs.map { |_, job| build_attributes(job[:name]) } -- cgit v1.2.3 From f52c61f846ce68b5d3517be3a0761773775dce29 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 18 Sep 2017 14:35:31 +0200 Subject: Do not initialize YAML processor with project full path --- lib/gitlab/ci/yaml_processor.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib/gitlab') diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index da39a28df4c..0bd78b03448 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -5,12 +5,11 @@ module Gitlab include Gitlab::Ci::Config::Entry::LegacyValidationHelpers - attr_reader :path, :cache, :stages, :jobs + attr_reader :cache, :stages, :jobs - def initialize(config, path = nil) + def initialize(config) @ci_config = Gitlab::Ci::Config.new(config) @config = @ci_config.to_hash - @path = path unless @ci_config.valid? raise ValidationError, @ci_config.errors.first -- cgit v1.2.3 From f028718641517a518f25c406afc71c4e54c914d3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 18 Sep 2017 14:39:09 +0200 Subject: Raise exception when initializing unknown policy --- lib/gitlab/ci/build/policy.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'lib/gitlab') diff --git a/lib/gitlab/ci/build/policy.rb b/lib/gitlab/ci/build/policy.rb index 6ab2f86c2e9..d10cc7802d4 100644 --- a/lib/gitlab/ci/build/policy.rb +++ b/lib/gitlab/ci/build/policy.rb @@ -4,11 +4,7 @@ module Gitlab module Policy def self.fabricate(specs) specifications = specs.to_h.map do |spec, value| - begin - self.const_get(spec.to_s.camelize).new(value) - rescue NameError - next - end + self.const_get(spec.to_s.camelize).new(value) end specifications.compact -- cgit v1.2.3