diff options
Diffstat (limited to 'lib/gitlab/ci/pipeline')
29 files changed, 321 insertions, 58 deletions
diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb new file mode 100644 index 00000000000..468f3bc4689 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class CancelPendingPipelines < Chain::Base + include Chain::Helpers + + def perform! + return unless project.auto_cancel_pending_pipelines? + + Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines) do |cancelables| + cancelables.find_each do |cancelable| + cancelable.auto_cancel_running(pipeline) + end + end + end + + def break? + false + end + + private + + # rubocop: disable CodeReuse/ActiveRecord + def auto_cancelable_pipelines + project.ci_pipelines + .where(ref: pipeline.ref) + .where.not(id: pipeline.same_family_pipeline_ids) + .where.not(sha: project.commit(pipeline.ref).try(:id)) + .alive_or_scheduled + .with_only_interruptible_builds + end + # rubocop: enable CodeReuse/ActiveRecord + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 74b28b181bc..dbaa6951e64 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -10,7 +10,7 @@ module Gitlab :trigger_request, :schedule, :merge_request, :external_pull_request, :ignore_skip_ci, :save_incompleted, :seeds_block, :variables_attributes, :push_options, - :chat_data, :allow_mirror_update, :bridge, :content, + :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, # These attributes are set by Chains during processing: :config_content, :config_processor, :stage_seeds ) do @@ -22,6 +22,8 @@ module Gitlab end end + alias_method :dry_run?, :dry_run + def branch_exists? strong_memoize(:is_branch) do project.repository.branch_exists?(ref) diff --git a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb index 3dd216b33d1..9954aedc4b7 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/parameter.rb @@ -12,7 +12,6 @@ module Gitlab def content strong_memoize(:content) do next unless command.content.present? - raise UnsupportedSourceError, "#{command.source} not a dangling build" unless command.dangling_build? command.content end diff --git a/lib/gitlab/ci/pipeline/chain/helpers.rb b/lib/gitlab/ci/pipeline/chain/helpers.rb index aba7dab508d..d7271df1694 100644 --- a/lib/gitlab/ci/pipeline/chain/helpers.rb +++ b/lib/gitlab/ci/pipeline/chain/helpers.rb @@ -6,13 +6,13 @@ module Gitlab module Chain module Helpers def error(message, config_error: false, drop_reason: nil) - if config_error && command.save_incompleted + if config_error drop_reason = :config_error pipeline.yaml_errors = message end pipeline.add_error_message(message) - pipeline.drop!(drop_reason) if drop_reason + pipeline.drop!(drop_reason) if drop_reason && persist_pipeline? # TODO: consider not to rely on AR errors directly as they can be # polluted with other unrelated errors (e.g. state machine) @@ -23,6 +23,10 @@ module Gitlab def warning(message) pipeline.add_warning_message(message) end + + def persist_pipeline? + command.save_incompleted && !pipeline.readonly? + end end end end diff --git a/lib/gitlab/ci/pipeline/chain/metrics.rb b/lib/gitlab/ci/pipeline/chain/metrics.rb new file mode 100644 index 00000000000..0d7449813b4 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/metrics.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class Metrics < Chain::Base + def perform! + counter.increment(source: @pipeline.source) + end + + def break? + false + end + + def counter + ::Gitlab::Ci::Pipeline::Metrics.new.pipelines_created_counter + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/pipeline/process.rb b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb new file mode 100644 index 00000000000..1eb7474e915 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/pipeline/process.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + module Pipeline + # After pipeline has been successfully created we can start processing it. + class Process < Chain::Base + def perform! + ::Ci::ProcessPipelineService + .new(@pipeline) + .execute + end + + def break? + false + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index 204c7725214..dc648568129 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -9,30 +9,21 @@ module Gitlab @pipeline = pipeline @command = command @sequence = sequence - @completed = [] @start = Time.now end def build! - @sequence.each do |chain| - step = chain.new(@pipeline, @command) + @sequence.each do |step_class| + step = step_class.new(@pipeline, @command) step.perform! break if step.break? - - @completed.push(step) end - @pipeline.tap do - yield @pipeline, self if block_given? - - @command.observe_creation_duration(Time.now - @start) - @command.observe_pipeline_size(@pipeline) - end - end + @command.observe_creation_duration(Time.now - @start) + @command.observe_pipeline_size(@pipeline) - def complete? - @completed.size == @sequence.size + @pipeline end end end diff --git a/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb b/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb new file mode 100644 index 00000000000..0e9add4ee74 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/stop_dry_run.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + # During the dry run we don't want to persist the pipeline and skip + # all the other steps that operate on a persisted context. + # This causes the chain to break at this point. + class StopDryRun < Chain::Base + def perform! + # no-op + end + + def break? + @command.dry_run? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 769d0dffd0b..8f1e690c081 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -34,7 +34,7 @@ module Gitlab end def allowed_to_write_ref? - access = Gitlab::UserAccess.new(current_user, project: project) + access = Gitlab::UserAccess.new(current_user, container: project) if @command.branch_exists? access.can_update_branch?(@command.ref) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb index 54a0e2ad9dd..422735bd104 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/and.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/and.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class And < Lexeme::Operator + class And < Lexeme::LogicalOperator PATTERN = /&&/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb index 7ebd2e25398..676857183cf 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/base.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/base.rb @@ -10,6 +10,10 @@ module Gitlab raise NotImplementedError end + def name + self.class.name.demodulize.underscore + end + def self.build(token) raise NotImplementedError end @@ -23,6 +27,10 @@ module Gitlab def self.pattern self::PATTERN end + + def self.consume?(lexeme) + lexeme && precedence >= lexeme.precedence + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb index 62f4c14f597..d35be12c996 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/equals.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class Equals < Lexeme::Operator + class Equals < Lexeme::LogicalOperator PATTERN = /==/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb new file mode 100644 index 00000000000..05d5043c06e --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/logical_operator.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class LogicalOperator < Lexeme::Operator + # This operator class is design to handle single operators that take two + # arguments. Expression::Parser was originally designed to read infix operators, + # and so the two operands are called "left" and "right" here. If we wish to + # implement an Operator that takes a greater or lesser number of arguments, a + # structural change or additional Operator superclass will likely be needed. + + def initialize(left, right) + raise OperatorError, 'Invalid left operand' unless left.respond_to? :evaluate + raise OperatorError, 'Invalid right operand' unless right.respond_to? :evaluate + + @left = left + @right = right + end + + def inspect + "#{name}(#{@left.inspect}, #{@right.inspect})" + end + + def self.type + :logical_operator + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb index f7b0720d4a9..4d65b914d8d 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/matches.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class Matches < Lexeme::Operator + class Matches < Lexeme::LogicalOperator PATTERN = /=~/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb index 8166bcd5730..64485a7e6b3 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_equals.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class NotEquals < Lexeme::Operator + class NotEquals < Lexeme::LogicalOperator PATTERN = /!=/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb index 02479ed28a4..29c5aa5d753 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/not_matches.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class NotMatches < Lexeme::Operator + class NotMatches < Lexeme::LogicalOperator PATTERN = /\!~/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb index be7258c201a..e7f7945532b 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/null.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/null.rb @@ -9,13 +9,17 @@ module Gitlab PATTERN = /null/.freeze def initialize(value = nil) - @value = nil + super end def evaluate(variables = {}) nil end + def inspect + 'null' + end + def self.build(_value) self.new end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb index 3ddab7800c8..a740c50c900 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/operator.rb @@ -6,24 +6,10 @@ module Gitlab module Expression module Lexeme class Operator < Lexeme::Base - # This operator class is design to handle single operators that take two - # arguments. Expression::Parser was originally designed to read infix operators, - # and so the two operands are called "left" and "right" here. If we wish to - # implement an Operator that takes a greater or lesser number of arguments, a - # structural change or additional Operator superclass will likely be needed. - OperatorError = Class.new(Expression::ExpressionError) - def initialize(left, right) - raise OperatorError, 'Invalid left operand' unless left.respond_to? :evaluate - raise OperatorError, 'Invalid right operand' unless right.respond_to? :evaluate - - @left = left - @right = right - end - def self.type - :operator + raise NotImplementedError end def self.precedence diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb index 807876f905a..c7d653ac859 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/or.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/or.rb @@ -5,7 +5,7 @@ module Gitlab module Pipeline module Expression module Lexeme - class Or < Lexeme::Operator + class Or < Lexeme::LogicalOperator PATTERN = /\|\|/.freeze def evaluate(variables = {}) diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb new file mode 100644 index 00000000000..b0ca26c9f5d --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_close.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class ParenthesisClose < Lexeme::Operator + PATTERN = /\)/.freeze + + def self.type + :parenthesis_close + end + + def self.precedence + 900 + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb new file mode 100644 index 00000000000..924fe0663ab --- /dev/null +++ b/lib/gitlab/ci/pipeline/expression/lexeme/parenthesis_open.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Expression + module Lexeme + class ParenthesisOpen < Lexeme::Operator + PATTERN = /\(/.freeze + + def self.type + :parenthesis_open + end + + def self.precedence + # Needs to be higher than `ParenthesisClose` and all other Lexemes + 901 + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 0212fa9d661..514241e8ae2 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -11,7 +11,7 @@ module Gitlab PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze def initialize(regexp) - @value = regexp.gsub(/\\\//, '/') + super(regexp.gsub(/\\\//, '/')) unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value) raise Lexer::SyntaxError, 'Invalid regular expression!' @@ -24,6 +24,10 @@ module Gitlab raise Expression::RuntimeError, 'Invalid regular expression!' end + def inspect + "/#{value}/" + end + def self.pattern PATTERN end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb index 2db2bf011f1..e90e764bcd9 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/string.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/string.rb @@ -9,13 +9,17 @@ module Gitlab PATTERN = /("(?<string>.*?)")|('(?<string>.*?)')/.freeze def initialize(value) - @value = value + super(value) end def evaluate(variables = {}) @value.to_s end + def inspect + @value.inspect + end + def self.build(string) new(string.match(PATTERN)[:string]) end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb index ef9ddb6cae9..6d872fee39d 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/value.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/value.rb @@ -9,6 +9,10 @@ module Gitlab def self.type :value end + + def initialize(value) + @value = value + end end end end diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb index 85c0899e4f6..11d2010909f 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb @@ -8,12 +8,12 @@ module Gitlab class Variable < Lexeme::Value PATTERN = /\$(?<name>\w+)/.freeze - def initialize(name) - @name = name + def evaluate(variables = {}) + variables.with_indifferent_access.fetch(@value, nil) end - def evaluate(variables = {}) - variables.with_indifferent_access.fetch(@name, nil) + def inspect + "$#{@value}" end def self.build(string) diff --git a/lib/gitlab/ci/pipeline/expression/lexer.rb b/lib/gitlab/ci/pipeline/expression/lexer.rb index 7d7582612f9..5b7365cb33b 100644 --- a/lib/gitlab/ci/pipeline/expression/lexer.rb +++ b/lib/gitlab/ci/pipeline/expression/lexer.rb @@ -10,6 +10,8 @@ module Gitlab SyntaxError = Class.new(Expression::ExpressionError) LEXEMES = [ + Expression::Lexeme::ParenthesisOpen, + Expression::Lexeme::ParenthesisClose, Expression::Lexeme::Variable, Expression::Lexeme::String, Expression::Lexeme::Pattern, @@ -22,6 +24,28 @@ module Gitlab Expression::Lexeme::Or ].freeze + # To be removed with `ci_if_parenthesis_enabled` + LEGACY_LEXEMES = [ + Expression::Lexeme::Variable, + Expression::Lexeme::String, + Expression::Lexeme::Pattern, + Expression::Lexeme::Null, + Expression::Lexeme::Equals, + Expression::Lexeme::Matches, + Expression::Lexeme::NotEquals, + Expression::Lexeme::NotMatches, + Expression::Lexeme::And, + Expression::Lexeme::Or + ].freeze + + def self.lexemes + if ::Gitlab::Ci::Features.ci_if_parenthesis_enabled? + LEXEMES + else + LEGACY_LEXEMES + end + end + MAX_TOKENS = 100 def initialize(statement, max_tokens: MAX_TOKENS) @@ -47,7 +71,7 @@ module Gitlab return tokens if @scanner.eos? - lexeme = LEXEMES.find do |type| + lexeme = self.class.lexemes.find do |type| type.scan(@scanner).tap do |token| tokens.push(token) if token.present? end diff --git a/lib/gitlab/ci/pipeline/expression/parser.rb b/lib/gitlab/ci/pipeline/expression/parser.rb index edb55edf356..27d7aa2f37e 100644 --- a/lib/gitlab/ci/pipeline/expression/parser.rb +++ b/lib/gitlab/ci/pipeline/expression/parser.rb @@ -15,11 +15,18 @@ module Gitlab def tree results = [] - tokens_rpn.each do |token| + tokens = + if ::Gitlab::Ci::Features.ci_if_parenthesis_enabled? + tokens_rpn + else + legacy_tokens_rpn + end + + tokens.each do |token| case token.type when :value results.push(token.build) - when :operator + when :logical_operator right_operand = results.pop left_operand = results.pop @@ -27,7 +34,7 @@ module Gitlab results.push(res) end else - raise ParseError, 'Unprocessable token found in parse tree' + raise ParseError, "Unprocessable token found in parse tree: #{token.type}" end end @@ -45,6 +52,7 @@ module Gitlab # Parse the expression into Reverse Polish Notation # (See: Shunting-yard algorithm) + # Taken from: https://en.wikipedia.org/wiki/Shunting-yard_algorithm#The_algorithm_in_detail def tokens_rpn output = [] operators = [] @@ -53,7 +61,34 @@ module Gitlab case token.type when :value output.push(token) - when :operator + when :logical_operator + output.push(operators.pop) while token.lexeme.consume?(operators.last&.lexeme) + + operators.push(token) + when :parenthesis_open + operators.push(token) + when :parenthesis_close + output.push(operators.pop) while token.lexeme.consume?(operators.last&.lexeme) + + raise ParseError, 'Unmatched parenthesis' unless operators.last + + operators.pop if operators.last.lexeme.type == :parenthesis_open + end + end + + output.concat(operators.reverse) + end + + # To be removed with `ci_if_parenthesis_enabled` + def legacy_tokens_rpn + output = [] + operators = [] + + @tokens.each do |token| + case token.type + when :value + output.push(token) + when :logical_operator if operators.any? && token.lexeme.precedence >= operators.last.lexeme.precedence output.push(operators.pop) end diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 649da745eea..db6cca27f1c 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -36,6 +36,15 @@ module Gitlab Gitlab::Metrics.counter(name, comment) end end + + def pipelines_created_counter + strong_memoize(:pipelines_created_count) do + name = :pipelines_created_total + comment = 'Counter of pipelines created' + + Gitlab::Metrics.counter(name, comment) + end + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 114a46ca9f6..3be3fa63b92 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -11,9 +11,7 @@ module Gitlab delegate :dig, to: :@seed_attributes - # When the `ci_dag_limit_needs` is enabled it uses the lower limit - LOW_NEEDS_LIMIT = 10 - HARD_NEEDS_LIMIT = 50 + DEFAULT_NEEDS_LIMIT = 10 def initialize(pipeline, attributes, previous_stages) @pipeline = pipeline @@ -142,10 +140,10 @@ module Gitlab end def max_needs_allowed - if Feature.enabled?(:ci_dag_limit_needs, @project, default_enabled: true) - LOW_NEEDS_LIMIT + if ::Gitlab::Ci::Features.ci_plan_needs_size_limit?(@pipeline.project) + @pipeline.project.actual_limits.ci_needs_size_limit else - HARD_NEEDS_LIMIT + DEFAULT_NEEDS_LIMIT end end |