diff options
author | Kamil Trzcinski <ayufan@ayufan.eu> | 2016-06-14 15:49:59 +0300 |
---|---|---|
committer | Kamil Trzcinski <ayufan@ayufan.eu> | 2016-06-14 15:49:59 +0300 |
commit | 3f5819fb7aa367ac302fe7d62857cfd604b5792f (patch) | |
tree | 68c1d6da2f7a24ba5e62d7077f4d30e8176882c8 /lib | |
parent | 1b62b86fdd1ad98e680c534f05fb32ff6e23fca4 (diff) | |
parent | 4b964011cfd6861403726248574f5926edc5d495 (diff) |
Merge remote-tracking branch 'origin/master' into artifacts-expire-date
Diffstat (limited to 'lib')
-rw-r--r-- | lib/banzai/pipeline/description_pipeline.rb | 17 | ||||
-rw-r--r-- | lib/ci/gitlab_ci_yaml_processor.rb | 41 | ||||
-rw-r--r-- | lib/gitlab/ci/config.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/ci/config/node/configurable.rb | 61 | ||||
-rw-r--r-- | lib/gitlab/ci/config/node/entry.rb | 77 | ||||
-rw-r--r-- | lib/gitlab/ci/config/node/factory.rb | 39 | ||||
-rw-r--r-- | lib/gitlab/ci/config/node/global.rb | 18 | ||||
-rw-r--r-- | lib/gitlab/ci/config/node/null.rb | 27 | ||||
-rw-r--r-- | lib/gitlab/ci/config/node/script.rb | 29 | ||||
-rw-r--r-- | lib/gitlab/ci/config/node/validation_helpers.rb | 34 | ||||
-rw-r--r-- | lib/gitlab/metrics/instrumentation.rb | 11 |
11 files changed, 321 insertions, 49 deletions
diff --git a/lib/banzai/pipeline/description_pipeline.rb b/lib/banzai/pipeline/description_pipeline.rb index f2395867658..042fb2e6e14 100644 --- a/lib/banzai/pipeline/description_pipeline.rb +++ b/lib/banzai/pipeline/description_pipeline.rb @@ -1,23 +1,16 @@ module Banzai module Pipeline class DescriptionPipeline < FullPipeline + WHITELIST = Banzai::Filter::SanitizationFilter::LIMITED.deep_dup.merge( + elements: Banzai::Filter::SanitizationFilter::LIMITED[:elements] - %w(pre code img ol ul li) + ) + def self.transform_context(context) super(context).merge( # SanitizationFilter - whitelist: whitelist + whitelist: WHITELIST ) end - - private - - def self.whitelist - # Descriptions are more heavily sanitized, allowing only a few elements. - # See http://git.io/vkuAN - whitelist = Banzai::Filter::SanitizationFilter::LIMITED - whitelist[:elements] -= %w(pre code img ol ul li) - - whitelist - end end end end diff --git a/lib/ci/gitlab_ci_yaml_processor.rb b/lib/ci/gitlab_ci_yaml_processor.rb index 76d84433cbe..e0b89cead06 100644 --- a/lib/ci/gitlab_ci_yaml_processor.rb +++ b/lib/ci/gitlab_ci_yaml_processor.rb @@ -2,6 +2,8 @@ module Ci class GitlabCiYamlProcessor class ValidationError < StandardError; end + include Gitlab::Ci::Config::Node::ValidationHelpers + DEFAULT_STAGES = %w(build test deploy) DEFAULT_STAGE = 'test' ALLOWED_YAML_KEYS = [:before_script, :after_script, :image, :services, :types, :stages, :variables, :cache] @@ -11,10 +13,12 @@ module Ci ALLOWED_CACHE_KEYS = [:key, :untracked, :paths] ALLOWED_ARTIFACTS_KEYS = [:name, :untracked, :paths, :when, :expire_in] - attr_reader :before_script, :after_script, :image, :services, :path, :cache + attr_reader :after_script, :image, :services, :path, :cache def initialize(config, path = nil) - @config = Gitlab::Ci::Config.new(config).to_hash + @ci_config = Gitlab::Ci::Config.new(config) + @config = @ci_config.to_hash + @path = path initial_parsing @@ -52,7 +56,6 @@ module Ci private def initial_parsing - @before_script = @config[:before_script] || [] @after_script = @config[:after_script] @image = @config[:image] @services = @config[:services] @@ -80,7 +83,7 @@ module Ci { stage_idx: stages.index(job[:stage]), stage: job[:stage], - commands: [job[:before_script] || @before_script, job[:script]].flatten.join("\n"), + commands: [job[:before_script] || [@ci_config.before_script], job[:script]].flatten.compact.join("\n"), tag_list: job[:tags] || [], name: name, only: job[:only], @@ -99,6 +102,10 @@ module Ci end def validate! + unless @ci_config.valid? + raise ValidationError, @ci_config.errors.first + end + validate_global! @jobs.each do |name, job| @@ -109,10 +116,6 @@ module Ci end def validate_global! - unless validate_array_of_strings(@before_script) - raise ValidationError, "before_script should be an array of strings" - end - unless @after_script.nil? || validate_array_of_strings(@after_script) raise ValidationError, "after_script should be an array of strings" end @@ -304,28 +307,6 @@ module Ci end end - def validate_duration(value) - value.is_a?(String) && ChronicDuration.parse(value) - rescue ChronicDuration::DurationParseError - false - end - - def validate_array_of_strings(values) - values.is_a?(Array) && values.all? { |value| validate_string(value) } - end - - def validate_variables(variables) - variables.is_a?(Hash) && variables.all? { |key, value| validate_string(key) && validate_string(value) } - end - - def validate_string(value) - value.is_a?(String) || value.is_a?(Symbol) - end - - def validate_boolean(value) - value.in?([true, false]) - end - def process?(only_params, except_params, ref, tag, trigger_request) if only_params.present? return false unless matching?(only_params, ref, tag, trigger_request) diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index ffe633d4b63..b48d3592f16 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -1,11 +1,21 @@ module Gitlab module Ci + ## + # Base GitLab CI Configuration facade + # class Config - class LoaderError < StandardError; end + delegate :valid?, :errors, to: :@global + + ## + # Temporary delegations that should be removed after refactoring + # + delegate :before_script, to: :@global def initialize(config) - loader = Loader.new(config) - @config = loader.load! + @config = Loader.new(config).load! + + @global = Node::Global.new(@config) + @global.process! end def to_hash diff --git a/lib/gitlab/ci/config/node/configurable.rb b/lib/gitlab/ci/config/node/configurable.rb new file mode 100644 index 00000000000..d60f87f3f94 --- /dev/null +++ b/lib/gitlab/ci/config/node/configurable.rb @@ -0,0 +1,61 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This mixin is responsible for adding DSL, which purpose is to + # simplifly process of adding child nodes. + # + # This can be used only if parent node is a configuration entry that + # holds a hash as a configuration value, for example: + # + # job: + # script: ... + # artifacts: ... + # + module Configurable + extend ActiveSupport::Concern + + def allowed_nodes + self.class.allowed_nodes || {} + end + + private + + def prevalidate! + unless @value.is_a?(Hash) + @errors << 'should be a configuration entry with hash value' + end + end + + def create_node(key, factory) + factory.with(value: @value[key]) + factory.nullify! unless @value.has_key?(key) + factory.create! + end + + class_methods do + def allowed_nodes + Hash[@allowed_nodes.map { |key, factory| [key, factory.dup] }] + end + + private + + def allow_node(symbol, entry_class, metadata) + factory = Node::Factory.new(entry_class) + .with(description: metadata[:description]) + + define_method(symbol) do + raise Entry::InvalidError unless valid? + + @nodes[symbol].try(:value) + end + + (@allowed_nodes ||= {}).merge!(symbol => factory) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/entry.rb b/lib/gitlab/ci/config/node/entry.rb new file mode 100644 index 00000000000..52758a962f3 --- /dev/null +++ b/lib/gitlab/ci/config/node/entry.rb @@ -0,0 +1,77 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Base abstract class for each configuration entry node. + # + class Entry + class InvalidError < StandardError; end + + attr_accessor :description + + def initialize(value) + @value = value + @nodes = {} + @errors = [] + + prevalidate! + end + + def process! + return if leaf? + return unless valid? + + compose! + + nodes.each(&:process!) + nodes.each(&:validate!) + end + + def nodes + @nodes.values + end + + def valid? + errors.none? + end + + def leaf? + allowed_nodes.none? + end + + def errors + @errors + nodes.map(&:errors).flatten + end + + def allowed_nodes + {} + end + + def validate! + raise NotImplementedError + end + + def value + raise NotImplementedError + end + + private + + def prevalidate! + end + + def compose! + allowed_nodes.each do |key, essence| + @nodes[key] = create_node(key, essence) + end + end + + def create_node(key, essence) + raise NotImplementedError + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/factory.rb b/lib/gitlab/ci/config/node/factory.rb new file mode 100644 index 00000000000..787ca006f5a --- /dev/null +++ b/lib/gitlab/ci/config/node/factory.rb @@ -0,0 +1,39 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Factory class responsible for fabricating node entry objects. + # + # It uses Fluent Interface pattern to set all necessary attributes. + # + class Factory + class InvalidFactory < StandardError; end + + def initialize(entry_class) + @entry_class = entry_class + @attributes = {} + end + + def with(attributes) + @attributes.merge!(attributes) + self + end + + def nullify! + @entry_class = Node::Null + self + end + + def create! + raise InvalidFactory unless @attributes.has_key?(:value) + + @entry_class.new(@attributes[:value]).tap do |entry| + entry.description = @attributes[:description] + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/global.rb b/lib/gitlab/ci/config/node/global.rb new file mode 100644 index 00000000000..044603423d5 --- /dev/null +++ b/lib/gitlab/ci/config/node/global.rb @@ -0,0 +1,18 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents a global entry - root node for entire + # GitLab CI Configuration file. + # + class Global < Entry + include Configurable + + allow_node :before_script, Script, + description: 'Script that will be executed before each job.' + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/null.rb b/lib/gitlab/ci/config/node/null.rb new file mode 100644 index 00000000000..4f590f6bec8 --- /dev/null +++ b/lib/gitlab/ci/config/node/null.rb @@ -0,0 +1,27 @@ +module Gitlab + module Ci + class Config + module Node + ## + # This class represents a configuration entry that is not being used + # in configuration file. + # + # This implements Null Object pattern. + # + class Null < Entry + def value + nil + end + + def validate! + nil + end + + def method_missing(*) + nil + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/script.rb b/lib/gitlab/ci/config/node/script.rb new file mode 100644 index 00000000000..5072bf0db7d --- /dev/null +++ b/lib/gitlab/ci/config/node/script.rb @@ -0,0 +1,29 @@ +module Gitlab + module Ci + class Config + module Node + ## + # Entry that represents a script. + # + # Each element in the value array is a command that will be executed + # by GitLab Runner. Currently we concatenate these commands with + # new line character as a separator, what is compatible with + # implementation in Runner. + # + class Script < Entry + include ValidationHelpers + + def value + @value.join("\n") + end + + def validate! + unless validate_array_of_strings(@value) + @errors << 'before_script should be an array of strings' + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/node/validation_helpers.rb b/lib/gitlab/ci/config/node/validation_helpers.rb new file mode 100644 index 00000000000..42ef60244ba --- /dev/null +++ b/lib/gitlab/ci/config/node/validation_helpers.rb @@ -0,0 +1,34 @@ +module Gitlab + module Ci + class Config + module Node + module ValidationHelpers + private + + def validate_duration(value) + value.is_a?(String) && ChronicDuration.parse(value) + rescue ChronicDuration::DurationParseError + false + end + + def validate_array_of_strings(values) + values.is_a?(Array) && values.all? { |value| validate_string(value) } + end + + def validate_variables(variables) + variables.is_a?(Hash) && + variables.all? { |key, value| validate_string(key) && validate_string(value) } + end + + def validate_string(value) + value.is_a?(String) || value.is_a?(Symbol) + end + + def validate_boolean(value) + value.in?([true, false]) + end + end + end + end + end +end diff --git a/lib/gitlab/metrics/instrumentation.rb b/lib/gitlab/metrics/instrumentation.rb index 0f115893a15..ad9ce3fa442 100644 --- a/lib/gitlab/metrics/instrumentation.rb +++ b/lib/gitlab/metrics/instrumentation.rb @@ -149,13 +149,16 @@ module Gitlab trans = Gitlab::Metrics::Instrumentation.transaction if trans - start = Time.now - retval = super - duration = (Time.now - start) * 1000.0 + start = Time.now + cpu_start = Gitlab::Metrics::System.cpu_time + retval = super + duration = (Time.now - start) * 1000.0 if duration >= Gitlab::Metrics.method_call_threshold + cpu_duration = Gitlab::Metrics::System.cpu_time - cpu_start + trans.add_metric(Gitlab::Metrics::Instrumentation::SERIES, - { duration: duration }, + { duration: duration, cpu_duration: cpu_duration }, method: #{label.inspect}) end |