From 2b9492a292d389d8390a9eca6a80e730ab7b6f1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Wed, 10 Apr 2019 11:04:51 +0200 Subject: Process at most 4 pipelines during push This adds a limitation that we will try to create pipeline for at most 4 first changes (branches and tags). This does not affect processing of Pipelines for Merge Requests, as each updated MR will have associated pipeline created. --- app/services/git/base_hooks_service.rb | 2 ++ app/workers/post_receive.rb | 27 +++++++------- .../limit-amount-of-created-pipelines.yml | 5 +++ doc/ci/yaml/README.md | 9 +++++ spec/workers/post_receive_spec.rb | 42 ++++++++++++++++++++-- 5 files changed, 70 insertions(+), 15 deletions(-) create mode 100644 changelogs/unreleased/limit-amount-of-created-pipelines.yml diff --git a/app/services/git/base_hooks_service.rb b/app/services/git/base_hooks_service.rb index fce4040e390..a8478e3a904 100644 --- a/app/services/git/base_hooks_service.rb +++ b/app/services/git/base_hooks_service.rb @@ -51,6 +51,8 @@ module Git end def create_pipelines + return unless params.fetch(:create_pipelines, true) + Ci::CreatePipelineService .new(project, current_user, push_data) .execute(:push, pipeline_options) diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index a5554f07699..337efa7919b 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -3,6 +3,8 @@ class PostReceive include ApplicationWorker + PIPELINE_PROCESS_LIMIT = 4 + def perform(gl_repository, identifier, changes, push_options = {}) project, repo_type = Gitlab::GlRepository.parse(gl_repository) @@ -36,23 +38,24 @@ class PostReceive return false end - post_received.changes_refs do |oldrev, newrev, ref| - if Gitlab::Git.tag_ref?(ref) - Git::TagPushService.new( - post_received.project, - @user, - oldrev: oldrev, - newrev: newrev, - ref: ref, - push_options: post_received.push_options).execute - elsif Gitlab::Git.branch_ref?(ref) - Git::BranchPushService.new( + post_received.enum_for(:changes_refs).with_index do |(oldrev, newrev, ref), index| + service_klass = + if Gitlab::Git.tag_ref?(ref) + Git::TagPushService + elsif Gitlab::Git.branch_ref?(ref) + Git::BranchPushService + end + + if service_klass + service_klass.new( post_received.project, @user, oldrev: oldrev, newrev: newrev, ref: ref, - push_options: post_received.push_options).execute + push_options: post_received.push_options, + create_pipelines: index < PIPELINE_PROCESS_LIMIT || Feature.enabled?(:git_push_create_all_pipelines, post_received.project) + ).execute end changes << Gitlab::DataBuilder::Repository.single_change(oldrev, newrev, ref) diff --git a/changelogs/unreleased/limit-amount-of-created-pipelines.yml b/changelogs/unreleased/limit-amount-of-created-pipelines.yml new file mode 100644 index 00000000000..51fdbb4d7ff --- /dev/null +++ b/changelogs/unreleased/limit-amount-of-created-pipelines.yml @@ -0,0 +1,5 @@ +--- +title: Process at most 4 pipelines during push +merge_request: 27205 +author: +type: performance diff --git a/doc/ci/yaml/README.md b/doc/ci/yaml/README.md index 5e44de13b51..79ab6ee6d62 100644 --- a/doc/ci/yaml/README.md +++ b/doc/ci/yaml/README.md @@ -2697,6 +2697,15 @@ Not to be confused with [`trigger`](#trigger-premium). [Read more in the triggers documentation.](../triggers/README.md) +## Processing Git pushes + +GitLab will create at most 4 branch and tags pipelines when +doing pushing multiple changes in single `git push` invocation. + +This limitation does not affect any of the updated Merge Request pipelines, +all updated Merge Requests will have a pipeline created when using +[pipelines for merge requests](../merge_request_pipelines/index.md). + ## Skipping jobs If your commit message contains `[ci skip]` or `[skip ci]`, using any diff --git a/spec/workers/post_receive_spec.rb b/spec/workers/post_receive_spec.rb index a3fe8fa4501..39f1beb4efa 100644 --- a/spec/workers/post_receive_spec.rb +++ b/spec/workers/post_receive_spec.rb @@ -99,11 +99,21 @@ describe PostReceive do end context "gitlab-ci.yml" do - let(:changes) { "123456 789012 refs/heads/feature\n654321 210987 refs/tags/tag" } + let(:changes) do + <<-EOF.strip_heredoc + 123456 789012 refs/heads/feature + 654321 210987 refs/tags/tag + 123456 789012 refs/heads/feature2 + 123458 789013 refs/heads/feature3 + 123459 789015 refs/heads/feature4 + EOF + end + + let(:changes_count) { changes.lines.count } subject { described_class.new.perform(gl_repository, key_id, base64_changes) } - context "creates a Ci::Pipeline for every change" do + context "with valid .gitlab-ci.yml" do before do stub_ci_pipeline_to_return_yaml_file @@ -116,7 +126,33 @@ describe PostReceive do .and_return(true) end - it { expect { subject }.to change { Ci::Pipeline.count }.by(2) } + context 'when git_push_create_all_pipelines is disabled' do + before do + stub_feature_flags(git_push_create_all_pipelines: false) + end + + it "creates pipeline for branches and tags" do + subject + + expect(Ci::Pipeline.pluck(:ref)).to contain_exactly("feature", "tag", "feature2", "feature3") + end + + it "creates exactly #{described_class::PIPELINE_PROCESS_LIMIT} pipelines" do + expect(changes_count).to be > described_class::PIPELINE_PROCESS_LIMIT + + expect { subject }.to change { Ci::Pipeline.count }.by(described_class::PIPELINE_PROCESS_LIMIT) + end + end + + context 'when git_push_create_all_pipelines is enabled' do + before do + stub_feature_flags(git_push_create_all_pipelines: true) + end + + it "creates all pipelines" do + expect { subject }.to change { Ci::Pipeline.count }.by(changes_count) + end + end end context "does not create a Ci::Pipeline" do -- cgit v1.2.3