From 251dd571dfc3e6261ed075ecf725dd98ee176b69 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 6 Jun 2016 11:05:15 +0200 Subject: Extract CI config validation helpers to mixin --- lib/ci/gitlab_ci_yaml_processor.rb | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) (limited to 'lib/ci') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 46a923161c8..e470ec56b79 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -2,6 +2,8 @@ module Ci class GitlabCiYamlProcessor class ValidationError < StandardError; end + include Gitlab::Ci::Config::ValidationHelpers + DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache] @@ -276,22 +278,6 @@ module Ci end end - def validate_array_of_strings(values) - values.is_a?(Array) && values.all? { |value| validate_string(value) } - end - - def validate_variables(variables) - variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } - end - - def validate_string(value) - value.is_a?(String) || value.is_a?(Symbol) - end - - def validate_boolean(value) - value.in?([true, false]) - end - def process?(only_params, except_params, ref, tag, trigger_request) if only_params.present? return false unless matching?(only_params, ref, tag, trigger_request) -- cgit v1.2.3 From 940763e0e72a7f71c6e60f2a1a848f8fe4afaf33 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Mon, 6 Jun 2016 12:23:27 +0200 Subject: Use CI config errors from new processor in legacy one --- lib/ci/gitlab_ci_yaml_processor.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'lib/ci') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index e470ec56b79..4bd2ac4f2db 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -14,7 +14,9 @@ module Ci attr_reader :before_script, :after_script, :image, :services, :path, :cache def initialize(config, path = nil) - @config = Gitlab::Ci::Config.new(config).to_hash + @ci_config = Gitlab::Ci::Config.new(config) + @config = @ci_config.to_hash + @path = path initial_parsing @@ -99,6 +101,10 @@ module Ci end def validate! + unless @ci_config.valid? + raise ValidationError, @ci_config.errors.first + end + validate_global! @jobs.each do |name, job| @@ -109,10 +115,6 @@ module Ci end def validate_global! - unless validate_array_of_strings(@before_script) - raise ValidationError, "before_script should be an array of strings" - end - unless @after_script.nil? || validate_array_of_strings(@after_script) raise ValidationError, "after_script should be an array of strings" end -- cgit v1.2.3 From df25c19699ba35682fd92da2b9c451bb4ba1c775 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 7 Jun 2016 12:58:32 +0200 Subject: Use Ci config validation helpers only where needed --- lib/ci/gitlab_ci_yaml_processor.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/ci') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 4bd2ac4f2db..c2b941a270a 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -2,7 +2,7 @@ module Ci class GitlabCiYamlProcessor class ValidationError < StandardError; end - include Gitlab::Ci::Config::ValidationHelpers + include Gitlab::Ci::Config::Node::ValidationHelpers DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' -- cgit v1.2.3 From 70bda3e89bc3828fc8771496ec6d61e41ac3d3ed Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 7 Jun 2016 14:23:47 +0200 Subject: Implement script in Ci config and use in legacy one --- lib/ci/gitlab_ci_yaml_processor.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/ci') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index c2b941a270a..0483e13b098 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -82,7 +82,7 @@ module Ci { stage_idx: stages.index(job[:stage]), stage: job[:stage], - commands: [job[:before_script] || @before_script, job[:script]].flatten.join("\n"), + commands: [job[:before_script] || [@ci_config.before_script], job[:script]].flatten.compact.join("\n"), tag_list: job[:tags] || [], name: name, only: job[:only], -- cgit v1.2.3 From cba266aabc60aeee64ac2eb7e76b3e9e7012bad4 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 8 Jun 2016 11:44:07 +0200 Subject: Remove old before_script from legacy Ci config --- lib/ci/gitlab_ci_yaml_processor.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib/ci') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index b37d231e893..c5a820563f0 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -11,7 +11,7 @@ module Ci :allow_failure, :type, :stage, :when, :artifacts, :cache, :dependencies, :before_script, :after_script, :variables] - attr_reader :before_script, :after_script, :image, :services, :path, :cache + attr_reader :after_script, :image, :services, :path, :cache def initialize(config, path = nil) @ci_config = Gitlab::Ci::Config.new(config) @@ -54,7 +54,6 @@ module Ci private def initial_parsing - @before_script = @config[:before_script] || [] @after_script = @config[:after_script] @image = @config[:image] @services = @config[:services] -- cgit v1.2.3 From 907c0e6796b69f9577c147dd489cf55748c749ac Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 10 Jun 2016 23:36:54 +0200 Subject: Added initial version of deployments --- lib/ci/gitlab_ci_yaml_processor.rb | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib/ci') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 130f5b0892e..5aacb59dc5c 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -7,7 +7,8 @@ module Ci ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache] ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache, - :dependencies, :before_script, :after_script, :variables] + :dependencies, :before_script, :after_script, :variables, + :environment] attr_reader :before_script, :after_script, :image, :services, :path, :cache @@ -85,6 +86,7 @@ module Ci except: job[:except], allow_failure: job[:allow_failure] || false, when: job[:when] || 'on_success', + environment: job[:environment], options: { image: job[:image] || @image, services: job[:services] || @services, @@ -203,6 +205,10 @@ module Ci if job[:when] && !job[:when].in?(%w(on_success on_failure always)) raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" end + + if job[:environment] && !validate_string(job[:environment]) + raise ValidationError, "#{name} job: environment should be a string" + end end def validate_job_script!(name, job) -- cgit v1.2.3 From 7a1b2e4f94e3e651d3264aa566a9056fe0f554e9 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 18 May 2016 15:28:46 -0500 Subject: Added when to artifacts --- lib/ci/gitlab_ci_yaml_processor.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'lib/ci') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 130f5b0892e..15d57a46eb0 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -8,6 +8,8 @@ module Ci ALLOWED_JOB_KEYS = [:tags, :script, :only, :except, :type, :image, :services, :allow_failure, :type, :stage, :when, :artifacts, :cache, :dependencies, :before_script, :after_script, :variables] + ALLOWED_CACHE_KEYS = [:key, :untracked, :paths] + ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when] attr_reader :before_script, :after_script, :image, :services, :path, :cache @@ -135,6 +137,12 @@ module Ci end def validate_global_cache! + @cache.keys.each do |key| + unless ALLOWED_CACHE_KEYS.include? key + raise ValidationError, "#{name} cache unknown parameter #{key}" + end + end + if @cache[:key] && !validate_string(@cache[:key]) raise ValidationError, "cache:key parameter should be a string" end @@ -233,6 +241,12 @@ module Ci end def validate_job_cache!(name, job) + job[:cache].keys.each do |key| + unless ALLOWED_CACHE_KEYS.include? key + raise ValidationError, "#{name} job: cache unknown parameter #{key}" + end + end + if job[:cache][:key] && !validate_string(job[:cache][:key]) raise ValidationError, "#{name} job: cache:key parameter should be a string" end @@ -247,6 +261,12 @@ module Ci end def validate_job_artifacts!(name, job) + job[:artifacts].keys.each do |key| + unless ALLOWED_ARTIFACTS_KEYS.include? key + raise ValidationError, "#{name} job: artifacts unknown parameter #{key}" + end + end + if job[:artifacts][:name] && !validate_string(job[:artifacts][:name]) raise ValidationError, "#{name} job: artifacts:name parameter should be a string" end @@ -258,6 +278,10 @@ module Ci if job[:artifacts][:paths] && !validate_array_of_strings(job[:artifacts][:paths]) raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings" end + + if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w(on_success on_failure always)) + raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always" + end end def validate_job_dependencies!(name, job) -- cgit v1.2.3 From aea4041ce96f18afea70da15af3cbe1be4fa1f94 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Wed, 18 May 2016 15:21:51 -0500 Subject: Allow to expire build artifacts --- lib/ci/api/builds.rb | 2 ++ lib/ci/gitlab_ci_yaml_processor.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'lib/ci') diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 607359769d1..54f5626c7d7 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -114,6 +114,7 @@ module Ci # id (required) - The ID of a build # token (required) - The build authorization token # file (required) - Artifacts file + # expire_in (optional) - Specify when artifacts should expire (ex. 7d) # Parameters (accelerated by GitLab Workhorse): # file.path - path to locally stored body (generated by Workhorse) # file.name - real filename as send in Content-Disposition @@ -145,6 +146,7 @@ module Ci build.artifacts_file = artifacts build.artifacts_metadata = metadata + build.artifacts_expire_at = Time.now + ChronicDuration.parse(params['expire_in']) if build.save present(build, with: Entities::BuildDetails) diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 15d57a46eb0..b1297565ebe 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -9,7 +9,7 @@ module Ci :allow_failure, :type, :stage, :when, :artifacts, :cache, :dependencies, :before_script, :after_script, :variables] ALLOWED_CACHE_KEYS = [:key, :untracked, :paths] - ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when] + ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in] attr_reader :before_script, :after_script, :image, :services, :path, :cache -- cgit v1.2.3 From 1501940ee0452f01acc5a228df17928e2f91cf39 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 10 Jun 2016 14:26:12 +0200 Subject: Validate artifacts:expire_in in yaml processor --- lib/ci/gitlab_ci_yaml_processor.rb | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'lib/ci') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index b1297565ebe..88fa079f30d 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -282,6 +282,10 @@ module Ci if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w(on_success on_failure always)) raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always" end + + if job[:artifacts][:expire_in] && !validate_duration(job[:artifacts][:expire_in]) + raise ValidationError, "#{name} job: artifacts:expire_in parameter should be a duration" + end end def validate_job_dependencies!(name, job) @@ -300,6 +304,12 @@ module Ci end end + def validate_duration(value) + value.is_a?(String) && ChronicDuration.parse(value) + rescue ChronicDuration::DurationParseError + false + end + def validate_array_of_strings(values) values.is_a?(Array) && values.all? { |value| validate_string(value) } end -- cgit v1.2.3 From 86800bf51aec25eef970eac82838bcba087703f8 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 10 Jun 2016 14:26:31 +0200 Subject: Support expiration date in CI API when uploading artifacts --- lib/ci/api/builds.rb | 2 +- lib/ci/api/entities.rb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/ci') diff --git a/lib/ci/api/builds.rb b/lib/ci/api/builds.rb index 54f5626c7d7..9f270f7b387 100644 --- a/lib/ci/api/builds.rb +++ b/lib/ci/api/builds.rb @@ -146,7 +146,7 @@ module Ci build.artifacts_file = artifacts build.artifacts_metadata = metadata - build.artifacts_expire_at = Time.now + ChronicDuration.parse(params['expire_in']) + build.artifacts_expire_in = params['expire_in'] if build.save present(build, with: Entities::BuildDetails) diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index a902ced35d7..352d92e7cc0 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -29,6 +29,7 @@ module Ci expose :before_sha expose :allow_git_fetch expose :token + expose :artifacts_expire_at, if: lambda { |build, opts| build.artifacts? } expose :options do |model| model.options -- cgit v1.2.3 From d23b91b0d9b8db16801872c49a1fb1d3be3a7144 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 10 Jun 2016 21:25:48 +0200 Subject: Improve after review --- lib/ci/gitlab_ci_yaml_processor.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/ci') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 88fa079f30d..76d84433cbe 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -208,7 +208,7 @@ module Ci raise ValidationError, "#{name} job: allow_failure parameter should be an boolean" end - if job[:when] && !job[:when].in?(%w(on_success on_failure always)) + if job[:when] && !job[:when].in?(%w[on_success on_failure always]) raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" end end @@ -279,7 +279,7 @@ module Ci raise ValidationError, "#{name} job: artifacts:paths parameter should be an array of strings" end - if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w(on_success on_failure always)) + if job[:artifacts][:when] && !job[:artifacts][:when].in?(%w[on_success on_failure always]) raise ValidationError, "#{name} job: artifacts:when parameter should be on_success, on_failure or always" end -- cgit v1.2.3 From 421be01dabb13cd1f45d0118b4e1be9d33baef61 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Fri, 10 Jun 2016 21:45:06 +0200 Subject: Improve design based on review --- lib/ci/api/entities.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/ci') diff --git a/lib/ci/api/entities.rb b/lib/ci/api/entities.rb index 352d92e7cc0..3f5bdaba3f5 100644 --- a/lib/ci/api/entities.rb +++ b/lib/ci/api/entities.rb @@ -20,7 +20,7 @@ module Ci expose :name, :token, :stage expose :project_id expose :project_name - expose :artifacts_file, using: ArtifactFile, if: lambda { |build, opts| build.artifacts? } + expose :artifacts_file, using: ArtifactFile, if: ->(build, _) { build.artifacts? } end class BuildDetails < Build @@ -29,7 +29,7 @@ module Ci expose :before_sha expose :allow_git_fetch expose :token - expose :artifacts_expire_at, if: lambda { |build, opts| build.artifacts? } + expose :artifacts_expire_at, if: ->(build, _) { build.artifacts? } expose :options do |model| model.options -- cgit v1.2.3 From e8f09f02bf8b0053f276a8e5ce0bdd18c621a1a3 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 14 Jun 2016 13:04:21 +0200 Subject: Validate environment name with regex --- lib/ci/gitlab_ci_yaml_processor.rb | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib/ci') diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 66f1bcea4ff..b19ce4aaff9 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -214,8 +214,8 @@ module Ci raise ValidationError, "#{name} job: when parameter should be on_success, on_failure or always" end - if job[:environment] && !validate_string(job[:environment]) - raise ValidationError, "#{name} job: environment should be a string" + if job[:environment] && !validate_environment(job[:environment]) + raise ValidationError, "#{name} job: environment parameter #{Gitlab::Regex.environment_name_regex_message}" end end @@ -322,6 +322,10 @@ module Ci value.in?([true, false]) end + def validate_environment(value) + value.is_a?(String) && value =~ Gitlab::Regex.environment_name_regex + end + def process?(only_params, except_params, ref, tag, trigger_request) if only_params.present? return false unless matching?(only_params, ref, tag, trigger_request) -- cgit v1.2.3