diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-02-18 13:34:06 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-02-18 13:34:06 +0300 |
commit | 859a6fb938bb9ee2a317c46dfa4fcc1af49608f0 (patch) | |
tree | d7f2700abe6b4ffcb2dcfc80631b2d87d0609239 /lib/gitlab/changelog | |
parent | 446d496a6d000c73a304be52587cd9bbc7493136 (diff) |
Add latest changes from gitlab-org/gitlab@13-9-stable-eev13.9.0-rc42
Diffstat (limited to 'lib/gitlab/changelog')
-rw-r--r-- | lib/gitlab/changelog/ast.rb | 157 | ||||
-rw-r--r-- | lib/gitlab/changelog/committer.rb | 69 | ||||
-rw-r--r-- | lib/gitlab/changelog/config.rb | 72 | ||||
-rw-r--r-- | lib/gitlab/changelog/error.rb | 8 | ||||
-rw-r--r-- | lib/gitlab/changelog/eval_state.rb | 26 | ||||
-rw-r--r-- | lib/gitlab/changelog/generator.rb | 59 | ||||
-rw-r--r-- | lib/gitlab/changelog/parser.rb | 176 | ||||
-rw-r--r-- | lib/gitlab/changelog/release.rb | 102 | ||||
-rw-r--r-- | lib/gitlab/changelog/template.tpl | 15 |
9 files changed, 684 insertions, 0 deletions
diff --git a/lib/gitlab/changelog/ast.rb b/lib/gitlab/changelog/ast.rb new file mode 100644 index 00000000000..2c787d396f5 --- /dev/null +++ b/lib/gitlab/changelog/ast.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # AST nodes to evaluate when rendering a template. + # + # Evaluating an AST is done by walking over the nodes and calling + # `evaluate`. This method takes two arguments: + # + # 1. An instance of `EvalState`, used for tracking data such as the number + # of nested loops. + # 2. An object used as the data for the current scope. This can be an Array, + # Hash, String, or something else. It's up to the AST node to determine + # what to do with it. + # + # While tree walking interpreters (such as implemented here) aren't usually + # the fastest type of interpreter, they are: + # + # 1. Fast enough for our use case + # 2. Easy to implement and maintain + # + # In addition, our AST interpreter doesn't allow for arbitrary code + # execution, unlike existing template engines such as Mustache + # (https://github.com/mustache/mustache/issues/244) or ERB. + # + # Our interpreter also takes care of limiting the number of nested loops. + # And unlike Liquid, our interpreter is much smaller and thus has a smaller + # attack surface. Liquid isn't without its share of issues, such as + # https://github.com/Shopify/liquid/pull/1071. + # + # We also evaluated using Handlebars using the project + # https://github.com/SmartBear/ruby-handlebars. Sadly, this implementation + # of Handlebars doesn't support control of whitespace + # (https://github.com/SmartBear/ruby-handlebars/issues/37), and the project + # didn't appear to be maintained that much. + # + # This doesn't mean these template engines aren't good, instead it means + # they won't work for our use case. For more information, refer to the + # comment https://gitlab.com/gitlab-org/gitlab/-/merge_requests/50063#note_469293322. + module AST + # An identifier in a selector. + Identifier = Struct.new(:name) do + def evaluate(state, data) + return data if name == 'it' + + data[name] if data.is_a?(Hash) + end + end + + # An integer used in a selector. + Integer = Struct.new(:value) do + def evaluate(state, data) + data[value] if data.is_a?(Array) + end + end + + # A selector used for loading a value. + Selector = Struct.new(:steps) do + def evaluate(state, data) + steps.reduce(data) do |current, step| + break if current.nil? + + step.evaluate(state, current) + end + end + end + + # A tag used for displaying a value in the output. + Variable = Struct.new(:selector) do + def evaluate(state, data) + selector.evaluate(state, data).to_s + end + end + + # A collection of zero or more expressions. + Expressions = Struct.new(:nodes) do + def evaluate(state, data) + nodes.map { |node| node.evaluate(state, data) }.join('') + end + end + + # A single text node. + Text = Struct.new(:text) do + def evaluate(*) + text + end + end + + # An `if` expression, with an optional `else` clause. + If = Struct.new(:condition, :true_body, :false_body) do + def evaluate(state, data) + result = + if truthy?(condition.evaluate(state, data)) + true_body.evaluate(state, data) + elsif false_body + false_body.evaluate(state, data) + end + + result.to_s + end + + def truthy?(value) + # We treat empty collections and such as false, removing the need for + # some sort of `if length(x) > 0` expression. + value.respond_to?(:empty?) ? !value.empty? : !!value + end + end + + # An `each` expression. + Each = Struct.new(:collection, :body) do + def evaluate(state, data) + values = collection.evaluate(state, data) + + return '' unless values.respond_to?(:each) + + # While unlikely to happen, it's possible users attempt to nest many + # loops in order to negatively impact the GitLab instance. To make + # this more difficult, we limit the number of nested loops a user can + # create. + state.enter_loop do + values.map { |value| body.evaluate(state, value) }.join('') + end + end + end + + # A class for transforming a raw Parslet AST into a more structured/easier + # to work with AST. + # + # For more information about Parslet transformations, refer to the + # documentation at http://kschiess.github.io/parslet/transform.html. + class Transformer < Parslet::Transform + rule(ident: simple(:name)) { Identifier.new(name.to_s) } + rule(int: simple(:name)) { Integer.new(name.to_i) } + rule(text: simple(:text)) { Text.new(text.to_s) } + rule(exprs: subtree(:nodes)) { Expressions.new(nodes) } + rule(selector: sequence(:steps)) { Selector.new(steps) } + rule(selector: simple(:step)) { Selector.new([step]) } + rule(variable: simple(:selector)) { Variable.new(selector) } + rule(each: simple(:values), body: simple(:body)) do + Each.new(values, body) + end + + rule(if: simple(:cond), true_body: simple(:true_body)) do + If.new(cond, true_body) + end + + rule( + if: simple(:cond), + true_body: simple(:true_body), + false_body: simple(:false_body) + ) do + If.new(cond, true_body, false_body) + end + end + end + end +end diff --git a/lib/gitlab/changelog/committer.rb b/lib/gitlab/changelog/committer.rb new file mode 100644 index 00000000000..31661650eff --- /dev/null +++ b/lib/gitlab/changelog/committer.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A class used for committing a release's changelog to a Git repository. + class Committer + def initialize(project, user) + @project = project + @user = user + end + + # Commits a release's changelog to a file on a branch. + # + # The `release` argument is a `Gitlab::Changelog::Release` for which to + # update the changelog. + # + # The `file` argument specifies the path to commit the changes to. + # + # The `branch` argument specifies the branch to commit the changes on. + # + # The `message` argument specifies the commit message to use. + def commit(release:, file:, branch:, message:) + # When retrying, we need to reprocess the existing changelog from + # scratch, otherwise we may end up throwing away changes. As such, all + # the logic is contained within the retry block. + Retriable.retriable(on: Error) do + commit = Gitlab::Git::Commit.last_for_path( + @project.repository, + branch, + file, + literal_pathspec: true + ) + + content = blob_content(file, commit) + + # If the release has already been added (e.g. concurrently by another + # API call), we don't want to add it again. + break if content&.match?(release.header_start_pattern) + + service = Files::MultiService.new( + @project, + @user, + commit_message: message, + branch_name: branch, + start_branch: branch, + actions: [ + { + action: content ? 'update' : 'create', + content: Generator.new(content.to_s).add(release), + file_path: file, + last_commit_id: commit&.sha + } + ] + ) + + result = service.execute + + raise Error.new(result[:message]) if result[:status] != :success + end + end + + def blob_content(file, commit = nil) + return unless commit + + @project.repository.blob_at(commit.sha, file)&.data + end + end + end +end diff --git a/lib/gitlab/changelog/config.rb b/lib/gitlab/changelog/config.rb new file mode 100644 index 00000000000..105050936ce --- /dev/null +++ b/lib/gitlab/changelog/config.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # Configuration settings used when generating changelogs. + class Config + # When rendering changelog entries, authors are not included. + AUTHORS_NONE = 'none' + + # The path to the configuration file as stored in the project's Git + # repository. + FILE_PATH = '.gitlab/changelog_config.yml' + + # The default date format to use for formatting release dates. + DEFAULT_DATE_FORMAT = '%Y-%m-%d' + + # The default template to use for generating release sections. + DEFAULT_TEMPLATE = File.read(File.join(__dir__, 'template.tpl')) + + attr_accessor :date_format, :categories, :template + + def self.from_git(project) + if (yaml = project.repository.changelog_config) + from_hash(project, YAML.safe_load(yaml)) + else + new(project) + end + end + + def self.from_hash(project, hash) + config = new(project) + + if (date = hash['date_format']) + config.date_format = date + end + + if (template = hash['template']) + config.template = Parser.new.parse_and_transform(template) + end + + if (categories = hash['categories']) + if categories.is_a?(Hash) + config.categories = categories + else + raise Error, 'The "categories" configuration key must be a Hash' + end + end + + config + end + + def initialize(project) + @project = project + @date_format = DEFAULT_DATE_FORMAT + @template = Parser.new.parse_and_transform(DEFAULT_TEMPLATE) + @categories = {} + end + + def contributor?(user) + @project.team.contributor?(user) + end + + def category(name) + @categories[name] || name + end + + def format_date(date) + date.strftime(@date_format) + end + end + end +end diff --git a/lib/gitlab/changelog/error.rb b/lib/gitlab/changelog/error.rb new file mode 100644 index 00000000000..0bd886fbdb7 --- /dev/null +++ b/lib/gitlab/changelog/error.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # An error raised when a changelog couldn't be generated. + Error = Class.new(StandardError) + end +end diff --git a/lib/gitlab/changelog/eval_state.rb b/lib/gitlab/changelog/eval_state.rb new file mode 100644 index 00000000000..a0439df60cf --- /dev/null +++ b/lib/gitlab/changelog/eval_state.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A class for tracking state when evaluating a template + class EvalState + MAX_LOOPS = 4 + + def initialize + @loops = 0 + end + + def enter_loop + if @loops == MAX_LOOPS + raise Error, "You can only nest up to #{MAX_LOOPS} loops" + end + + @loops += 1 + retval = yield + @loops -= 1 + + retval + end + end + end +end diff --git a/lib/gitlab/changelog/generator.rb b/lib/gitlab/changelog/generator.rb new file mode 100644 index 00000000000..a80ca0728f9 --- /dev/null +++ b/lib/gitlab/changelog/generator.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # Parsing and generating of Markdown changelogs. + class Generator + # The regex used to parse a release header. + RELEASE_REGEX = + /^##\s+(?<version>#{Gitlab::Regex.unbounded_semver_regex})/.freeze + + # The `input` argument must be a `String` containing the existing + # changelog Markdown. If no changelog exists, this should be an empty + # `String`. + def initialize(input = '') + @lines = input.lines + @locations = {} + + @lines.each_with_index do |line, index| + matches = line.match(RELEASE_REGEX) + + next if !matches || !matches[:version] + + @locations[matches[:version]] = index + end + end + + # Generates the Markdown for the given release and returns the new + # changelog Markdown content. + # + # The `release` argument must be an instance of + # `Gitlab::Changelog::Release`. + def add(release) + versions = [release.version, *@locations.keys] + + VersionSorter.rsort!(versions) + + new_index = versions.index(release.version) + new_lines = @lines.dup + markdown = release.to_markdown + + if (insert_after = versions[new_index + 1]) + line_index = @locations[insert_after] + + new_lines.insert(line_index, markdown) + else + # When adding to the end of the changelog, the previous section only + # has a single newline, resulting in the release section title + # following it immediately. When this is the case, we insert an extra + # empty line to keep the changelog readable in its raw form. + new_lines.push("\n") if versions.length > 1 + new_lines.push(markdown.rstrip) + new_lines.push("\n") + end + + new_lines.join + end + end + end +end diff --git a/lib/gitlab/changelog/parser.rb b/lib/gitlab/changelog/parser.rb new file mode 100644 index 00000000000..a4c8da283cd --- /dev/null +++ b/lib/gitlab/changelog/parser.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A parser for the template syntax used for generating changelogs. + # + # As a quick primer on the template syntax, a basic template looks like + # this: + # + # {% each users %} + # Name: {{name}} + # Age: {{age}} + # + # {% if birthday %} + # This user is celebrating their birthday today! Yay! + # {% end %} + # {% end %} + # + # For more information, refer to the Parslet documentation found at + # http://kschiess.github.io/parslet/. + class Parser < Parslet::Parser + root(:exprs) + + rule(:exprs) do + ( + variable | if_expr | each_expr | escaped | text | newline + ).repeat.as(:exprs) + end + + rule(:space) { match('[ \\t]') } + rule(:whitespace) { match('\s').repeat } + rule(:lf) { str("\n") } + rule(:newline) { lf.as(:text) } + + # Escaped newlines are ignored, allowing the user to control the + # whitespace in the output. All other escape sequences are treated as + # literal text. + # + # For example, this: + # + # foo \ + # bar + # + # Is parsed into this: + # + # foo bar + rule(:escaped) do + backslash = str('\\') + + (backslash >> lf).ignore | (backslash >> chars).as(:text) + end + + # A sequence of regular characters, with the exception of newlines and + # escaped newlines. + rule(:chars) do + char = match("[^{\\\\\n]") + + # The rules here are such that we do treat single curly braces or + # non-opening tags (e.g. `{foo}`) as text, but not opening tags + # themselves (e.g. `{{`). + ( + char.repeat(1) | curly_open >> (curly_open | percent).absent? + ).repeat(1) + end + + rule(:text) { chars.as(:text) } + + # An integer, limited to 10 digits (= a 32 bits integer). + # + # The size is limited to prevents users from creating integers that are + # too large, as this may result in runtime errors. + rule(:integer) { match('\d').repeat(1, 10).as(:int) } + + # An identifier to look up in a data structure. + # + # We only support simple ASCII identifiers as we simply don't have a need + # for more complex identifiers (e.g. those containing multibyte + # characters). + rule(:ident) { match('[a-zA-Z_]').repeat(1).as(:ident) } + + # A selector is used for reading a value, consisting of one or more + # "steps". + # + # Examples: + # + # name + # users.0.name + # 0 + # it + rule(:selector) do + step = ident | integer + + whitespace >> + (step >> (str('.') >> step).repeat).as(:selector) >> + whitespace + end + + rule(:curly_open) { str('{') } + rule(:curly_close) { str('}') } + rule(:percent) { str('%') } + + # A variable tag. + # + # Examples: + # + # {{name}} + # {{users.0.name}} + rule(:variable) do + curly_open.repeat(2) >> selector.as(:variable) >> curly_close.repeat(2) + end + + rule(:expr_open) { curly_open >> percent >> whitespace } + rule(:expr_close) do + # Since whitespace control is important (as Markdown is whitespace + # sensitive), we default to stripping a newline that follows a %} tag. + # This is less annoying compared to having to opt-in to this behaviour. + whitespace >> percent >> curly_close >> lf.maybe.ignore + end + + rule(:end_tag) { expr_open >> str('end') >> expr_close } + + # An `if` expression, with an optional `else` clause. + # + # Examples: + # + # {% if foo %} + # yes + # {% end %} + # + # {% if foo %} + # yes + # {% else %} + # no + # {% end %} + rule(:if_expr) do + else_tag = + expr_open >> str('else') >> expr_close >> exprs.as(:false_body) + + expr_open >> + str('if') >> + space.repeat(1) >> + selector.as(:if) >> + expr_close >> + exprs.as(:true_body) >> + else_tag.maybe >> + end_tag + end + + # An `each` expression, used for iterating over collections. + # + # Example: + # + # {% each users %} + # * {{name}} + # {% end %} + rule(:each_expr) do + expr_open >> + str('each') >> + space.repeat(1) >> + selector.as(:each) >> + expr_close >> + exprs.as(:body) >> + end_tag + end + + def parse_and_transform(input) + AST::Transformer.new.apply(parse(input)) + rescue Parslet::ParseFailed => ex + # We raise a custom error so it's easier to catch different changelog + # related errors. In addition, this ensures the caller of this method + # doesn't depend on a Parslet specific error class. + raise Error.new("Failed to parse the template: #{ex.message}") + end + end + end +end diff --git a/lib/gitlab/changelog/release.rb b/lib/gitlab/changelog/release.rb new file mode 100644 index 00000000000..f2a01c2b0dc --- /dev/null +++ b/lib/gitlab/changelog/release.rb @@ -0,0 +1,102 @@ +# frozen_string_literal: true + +module Gitlab + module Changelog + # A release to add to a changelog. + class Release + attr_reader :version + + def initialize(version:, date:, config:) + @version = version + @date = date + @config = config + @entries = Hash.new { |h, k| h[k] = [] } + + # This ensures that entries are presented in the same order as the + # categories Hash in the user's configuration. + @config.categories.values.each do |category| + @entries[category] = [] + end + end + + def add_entry( + title:, + commit:, + category:, + author: nil, + merge_request: nil + ) + # When changing these fields, keep in mind that this needs to be + # backwards compatible. For example, you can't just remove a field as + # this will break the changelog generation process for existing users. + entry = { + 'title' => title, + 'commit' => { + 'reference' => commit.to_reference(full: true), + 'trailers' => commit.trailers + } + } + + if author + entry['author'] = { + 'reference' => author.to_reference(full: true), + 'contributor' => @config.contributor?(author) + } + end + + if merge_request + entry['merge_request'] = { + 'reference' => merge_request.to_reference(full: true) + } + end + + @entries[@config.category(category)] << entry + end + + def to_markdown + state = EvalState.new + data = { 'categories' => entries_for_template } + + # While not critical, we would like release sections to be separated by + # an empty line in the changelog; ensuring it's readable even in its + # raw form. + # + # Since it can be a bit tricky to get this right in a template, we + # enforce an empty line separator ourselves. + markdown = @config.template.evaluate(state, data).strip + + # The release header can't be changed using the Liquid template, as we + # need this to be in a known format. Without this restriction, we won't + # know where to insert a new release section in an existing changelog. + "## #{@version} (#{release_date})\n\n#{markdown}\n\n" + end + + def header_start_pattern + /^##\s*#{Regexp.escape(@version)}/ + end + + private + + def release_date + @config.format_date(@date) + end + + def entries_for_template + rows = [] + + @entries.each do |category, entries| + next if entries.empty? + + rows << { + 'title' => category, + 'count' => entries.length, + 'single_change' => entries.length == 1, + 'entries' => entries + } + end + + rows + end + end + end +end diff --git a/lib/gitlab/changelog/template.tpl b/lib/gitlab/changelog/template.tpl new file mode 100644 index 00000000000..584939dff51 --- /dev/null +++ b/lib/gitlab/changelog/template.tpl @@ -0,0 +1,15 @@ +{% if categories %} +{% each categories %} +### {{ title }} ({% if single_change %}1 change{% else %}{{ count }} changes{% end %}) + +{% each entries %} +- [{{ title }}]({{ commit.reference }})\ +{% if author.contributor %} by {{ author.reference }}{% end %}\ +{% if merge_request %} ([merge request]({{ merge_request.reference }})){% end %} + +{% end %} + +{% end %} +{% else %} +No changes. +{% end %} |