diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-19 14:50:12 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2019-09-19 14:50:12 +0300 |
commit | 6cd5b7dbfaa4ff630ecbbfe351a1faac5fc71a8d (patch) | |
tree | d3563b9f60936c18a02185e2e53b424bb1b83539 /scripts | |
parent | b3e0658cb1fbc7c8e7dd381467c656f2e675ee46 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'scripts')
-rw-r--r-- | scripts/ee_specific_check/ee_specific_check.rb | 470 | ||||
-rwxr-xr-x | scripts/static-analysis | 4 |
2 files changed, 0 insertions, 474 deletions
diff --git a/scripts/ee_specific_check/ee_specific_check.rb b/scripts/ee_specific_check/ee_specific_check.rb deleted file mode 100644 index cdb62f66e94..00000000000 --- a/scripts/ee_specific_check/ee_specific_check.rb +++ /dev/null @@ -1,470 +0,0 @@ -# frozen_string_literal: true -# rubocop: disable CodeReuse/ActiveRecord - -module EESpecificCheck - WHITELIST = [ - 'CHANGELOG-EE.md', - 'scripts/**/*', - 'vendor/assets/javascripts/jasmine-jquery.js', - '.gitlab-ci.yml', - '.gitlab/ci/rails.gitlab-ci.yml', - 'db/schema.rb', - 'locale/gitlab.pot' - ].freeze - - CompareBase = Struct.new(:ce_base, :ee_base, :ce_head, :ee_head) - GitStatus = Struct.new(:porcelain, :head) - - module_function - - def git_version - say run_git_command('--version') - end - - def say(message) - warn "\n#{message}", "\n" # puts would eat trailing newline - end - - def find_compare_base - git_clean - - ce_fetch_head = fetch_remote_ce_branch - ee_fetch_head = head_commit_sha - ce_fetch_base = find_merge_base('canonical-ce/master', ce_fetch_head) - ee_fetch_base = find_merge_base('canonical-ee/master', 'HEAD') - ce_merge_base = find_merge_base(ce_fetch_head, ee_fetch_head) - - ce_updated_head = - find_ce_compare_head(ce_fetch_head, ce_fetch_base, ce_merge_base) - - CompareBase.new( - ce_merge_base, ee_fetch_base, ce_updated_head, ee_fetch_head) - end - - def setup_canonical_remotes - run_git_command( - "remote add canonical-ee https://gitlab.com/gitlab-org/gitlab.git", - "remote add canonical-ce https://gitlab.com/gitlab-org/gitlab-foss.git", - "fetch canonical-ee master --quiet --depth=9999", - "fetch canonical-ce master --quiet --depth=9999") - end - - def fetch_remote_ce_branch - setup_canonical_remotes - - remote_to_fetch, branch_to_fetch = find_remote_ce_branch - - run_git_command("fetch #{remote_to_fetch} #{branch_to_fetch} --quiet --depth=9999") - - "#{remote_to_fetch}/#{branch_to_fetch}" - end - - def find_merge_base(left, right) - merge_base = run_git_command("merge-base #{left} #{right}") - - return merge_base unless merge_base.empty? - - say <<~MESSAGE - 💥 Unfortunately we cannot find the merge-base for #{left} and #{right}, - 💥 and we'll try to fix that in: - https://gitlab.com/gitlab-org/gitlab/issues/9120 - - 💥 Before that, please run this job locally as a workaround: - - ./scripts/ee-specific-lines-check - - 💥 And paste the result as a discussion to show it to the maintainer. - 💥 If you have any questions, please ping @godfat to investigate and - 💥 clarify. - MESSAGE - - exit(253) - end - - def find_ce_compare_head(ce_fetch_head, ce_fetch_base, ce_merge_base) - if git_ancestor?(ce_merge_base, ce_fetch_base) - say("CE is ahead of EE, finding backward CE head") - find_backward_ce_head(ce_fetch_head, ce_fetch_base, ce_merge_base) - else - say("CE is behind of EE, finding forward CE head") - find_forward_ce_head(ce_merge_base, ce_fetch_head) - end - end - - def git_ancestor?(ancestor, descendant) - run_git_command( - "merge-base --is-ancestor #{ancestor} #{descendant} && echo y") == 'y' - end - - def find_backward_ce_head(ce_fetch_head, ce_fetch_base, ce_merge_base) - if ce_fetch_head.start_with?('canonical-ce') # No specific CE branch - say("No CE branch found, using merge base directly") - ce_merge_base - elsif ce_fetch_base == ce_merge_base # Up-to-date, no rebase needed - say("EE is up-to-date with CE, using #{ce_fetch_head} directly") - ce_fetch_head - else - say("Performing rebase to remove commits in CE haven't merged into EE") - checkout_and_rebase(ce_merge_base, ce_fetch_base, ce_fetch_head) - end - end - - def find_forward_ce_head(ce_merge_base, ce_fetch_head) - say("Performing merge with CE master for CE branch #{ce_fetch_head}") - with_detached_head(ce_fetch_head) do - run_git_command("merge #{ce_merge_base} -s recursive -X patience -m 'ee-specific-auto-merge'") - - status = git_status - - if status.porcelain == '' - status.head - else - diff = run_git_command("diff") - run_git_command("merge --abort") - - say <<~MESSAGE - 💥 Git status not clean! This means there's a conflict in - 💥 #{ce_fetch_head} with canonical-ce/master. Please resolve - 💥 the conflict from CE master and retry this job. - - ⚠️ Git diff: - - #{diff} - MESSAGE - - exit(254) - end - end - end - - # We rebase onto the commit which is the latest commit presented in both - # CE and EE, i.e. ce_merge_base, cutting off commits aren't merged into - # EE yet. Here's an example: - # - # * o: Relevant commits - # * x: Irrelevant commits - # * !: Commits we want to cut off from CE branch - # - # ^-> o CE branch (ce_fetch_head) - # / (ce_fetch_base) - # o -> o -> ! -> x CE master - # v (ce_merge_base) - # o -> o -> o -> x EE master - # \ (ee_fetch_base) - # v-> o EE branch - # - # We want to rebase above into this: (we only change the connection) - # - # -> - -> o CE branch (ce_fetch_head) - # / (ce_fetch_base) - # o -> o -> ! -> x CE master - # v (ce_merge_base) - # o -> o -> o -> x EE master - # \ (ee_fetch_base) - # v-> o EE branch - # - # Therefore we rebase onto ce_merge_base, which is based off CE master, - # for the CE branch (ce_fetch_head), effective remove the commit marked - # as ! in the graph for CE branch. We need to remove it because it's not - # merged into EE yet, therefore won't be available in the EE branch. - # - # After rebase is done, then we could compare against - # ce_merge_base..ee_fetch_base along with ce_fetch_head..HEAD (EE branch) - # where ce_merge_base..ee_fetch_base is the update-to-date - # CE/EE difference and ce_fetch_head..HEAD is the changes we made in - # CE and EE branches. - def checkout_and_rebase(new_base, old_base, target_head) - with_detached_head(target_head) do - run_git_command("rebase --onto #{new_base} #{old_base} #{target_head}") - - status = git_status - - if status.porcelain == '' - status.head - else - diff = run_git_command("diff") - run_git_command("rebase --abort") - - say <<~MESSAGE - 💥 Git status is not clean! This means the CE branch has or had a - 💥 conflict with CE master, and we cannot resolve this in an - 💥 automatic way. - 💥 - 💥 Please rebase #{target_head} with CE master. - 💥 - 💥 For more details, please read: - 💥 https://gitlab.com/gitlab-org/gitlab/issues/6038#note_86862115 - 💥 - 💥 Git diff: - - #{diff} - MESSAGE - - exit(255) - end - end - end - - def with_detached_head(target_head) - # So that we could switch back. CI sometimes doesn't have the branch, - # so we don't use current_branch here - head = current_head - - # Use detached HEAD so that we don't update HEAD - run_git_command("checkout -f #{target_head}") - git_clean - - yield - ensure # ensure would still run if we call exit, don't worry - # Make sure to switch back - run_git_command("checkout -f #{head}") - git_clean - end - - def head_commit_sha - run_git_command("rev-parse HEAD") - end - - def git_status - GitStatus.new( - run_git_command("status --porcelain"), - head_commit_sha - ) - end - - def git_clean - # We're still seeing errors not ignoring knapsack/ and rspec_flaky/ - # Instead of waiting that populate over all the branches, we could - # just remove untracked files anyway, only on CI of course in case - # we're wiping people's data! - # See https://gitlab.com/gitlab-org/gitlab/issues/5912 - # Also see https://gitlab.com/gitlab-org/gitlab/-/jobs/68194333 - run_git_command('clean -fd') if ENV['CI'] - end - - def remove_remotes - run_git_command( - "remote remove canonical-ee", - "remote remove canonical-ce", - "remote remove target-ce") - end - - def updated_diff_numstat(from, to) - scan_diff_numstat( - run_git_command("diff #{from}..#{to} --numstat -- . ':!ee' ':!qa/qa/ee' ':!qa/qa/ee.rb' ':!qa/qa/specs/features/ee'")) - end - - def find_remote_ce_branch - branch_to_fetch = matching_ce_refs.first - - if branch_to_fetch - say "💪 We found the branch '#{branch_to_fetch}' in the #{ce_repo_url} repository. We will fetch it." - - run_git_command("remote add target-ce #{ce_repo_url}") - - ['target-ce', branch_to_fetch] - else - say <<~MESSAGE - ⚠️ We did not find a branch that would match the current '#{current_branch}' branch in the #{ce_repo_url} repository. We will fetch 'master' instead. - ℹ️ If you have a CE branch for the current branch, make sure that its name includes '#{minimal_ce_branch_name}'. - MESSAGE - - %w[canonical-ce master] - end - end - - def ce_repo_url - @ce_repo_url ||= - begin - repo_url = ENV.fetch('CI_REPOSITORY_URL', 'https://gitlab.com/gitlab-org/gitlab-foss.git') - # This workaround can be removed once we rename the dev CE project - # https://gitlab.com/gitlab-org/gitlab-foss/issues/59107 - project_name = repo_url =~ /dev\.gitlab\.org/ ? 'gitlabhq' : 'gitlab-ce' - - repo_url.sub('gitlab-ee', project_name) - end - end - - def current_head - @current_head ||= ENV.fetch('CI_COMMIT_SHA', current_branch) - end - - def current_branch - @current_branch ||= ENV.fetch('CI_COMMIT_REF_NAME', `git rev-parse --abbrev-ref HEAD`).strip - end - - def minimal_ce_branch_name - @minimal_ce_branch_name ||= current_branch.sub(/(\Aee\-|\-ee\z)/, '') - end - - def matching_ce_refs - @matching_ce_refs ||= - run_git_command("ls-remote #{ce_repo_url} \"*#{minimal_ce_branch_name}*\"") - .scan(%r{(?<=refs/heads/|refs/tags/).+}) - .select { |branch| branch.match?(/\b#{minimal_ce_branch_name}\b/i) } - .sort_by(&:size) - end - - def scan_diff_numstat(numstat) - numstat.scan(/(\d+)\s+(\d+)\s+(.+)/) - .each_with_object(Hash.new(0)) do |(added, deleted, file), result| - result[file] = added.to_i + deleted.to_i - end - end - - def run_git_command(*commands) - cmds = commands.map { |cmd| "git #{cmd}" } - - output = run_command(*cmds) - - if commands.size == 1 - output.first - else - output - end - end - - def run_command(*commands) - commands.map do |cmd| - warn "=> Running `#{cmd}`" - - `#{cmd}`.strip - end - end -end - -if $0 == __FILE__ - require 'rspec/autorun' - - RSpec.describe EESpecificCheck do - subject { Class.new { include EESpecificCheck }.new } - - before do - allow(subject).to receive(:warn) - - EESpecificCheck.private_instance_methods.each do |name| - subject.class.__send__(:public, name) # rubocop:disable GitlabSecurity/PublicSend - end - end - - describe '.run_git_command' do - it 'returns the single output when there is a single command' do - output = subject.run_git_command('status') - - expect(output).to be_kind_of(String) - expect(subject).to have_received(:warn).with(/git status/) - end - - it 'returns an array of output for more commands' do - output = subject.run_git_command('status', 'help') - - expect(output).to all(be_a(String)) - expect(subject).to have_received(:warn).with(/git status/) - expect(subject).to have_received(:warn).with(/git help/) - end - end - - describe '.find_merge_base' do - context 'when it cannot find the merge base' do - before do - allow(subject).to receive(:say) - allow(subject).to receive(:exit) - - expect(subject).to receive(:run_git_command).and_return('') - end - - it 'calls exit(253) to fail the job and ask run it locally' do - subject.find_merge_base('master', 'HEAD') - - expect(subject).to have_received(:say) - .with(Regexp.union('./scripts/ee-specific-lines-check')) - expect(subject).to have_received(:exit) - .with(253) - end - end - - context 'when it found the merge base' do - before do - expect(subject).to receive(:run_git_command).and_return('deadbeef') - end - - it 'returns the found merge base' do - output = subject.find_merge_base('master', 'HEAD') - - expect(output).to eq('deadbeef') - end - end - end - - describe '.matching_ce_refs' do - before do - expect(subject).to receive(:current_branch).and_return(ee_branch) - expect(subject).to receive(:run_git_command) - .and_return(ls_remote_output) - end - - describe 'simple cases' do - let(:ls_remote_output) do - <<~OUTPUT - d6602ec5194c87b0fc87103ca4d67251c76f233a\trefs/tags/v9 - f25a265a342aed6041ab0cc484224d9ca54b6f41\trefs/tags/v9.12 - c5db5456ae3b0873fc659c19fafdde22313cc441\trefs/tags/v9.123 - 0918385dbd9656cab0d1d81ba7453d49bbc16250\trefs/heads/v9.x - 28862662b749fe981386814e2dba87b0e72c1eab\trefs/remotes/remote_mirror_3059/v9-to-fix-http-case-problems - 5e3496802098c86050c5b463507f3a68a83a9f02\trefs/remotes/remote_mirror_3059/29036-use-slack-service-v9 - OUTPUT - end - - context 'with a ee- prefix' do - let(:ee_branch) { 'ee-v9' } - - it 'sorts by matching size' do - expect(subject.matching_ce_refs).to eq(%w[v9 v9.x v9.12 v9.123]) - end - end - - context 'with a -ee suffix' do - let(:ee_branch) { 'v9-ee' } - - it 'sorts by matching size' do - expect(subject.matching_ce_refs).to eq(%w[v9 v9.x v9.12 v9.123]) - end - end - end - - describe 'with ambiguous branch name' do - let(:ls_remote_output) do - <<~OUTPUT - 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/feature/sm/35954-expand-kubernetesservice-to-use-username-password - 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ce-to-ee-231 - 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ce-to-ee-2 - 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ce-to-1 - 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ee-to-ce-123 - 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/ee-to-ce-12 - 954d7119384c9f2a3c862bac97beb641eb8755d6\trefs/heads/to-ce-1 - 28862662b749fe981386814e2dba87b0e72c1eab\trefs/remotes/remote_mirror_3059/27056-upgrade-vue-resource-to-1-0-3-to-fix-http-case-problems - 5e3496802098c86050c5b463507f3a68a83a9f02\trefs/remotes/remote_mirror_3059/29036-use-slack-service-to-notify-of-failed-pipelines - OUTPUT - end - - context 'with a ee- prefix' do - let(:ee_branch) { 'ee-to-ce' } - let(:minimal_ce_branch) { 'to-ce' } - - it 'sorts by matching size' do - expect(subject.matching_ce_refs).to eq(%w[to-ce-1 ee-to-ce-12 ee-to-ce-123]) - end - end - - context 'with a -ee suffix' do - let(:ee_branch) { 'ce-to-ee' } - let(:minimal_ce_branch) { 'ce-to' } - - it 'sorts by matching size' do - expect(subject.matching_ce_refs).to eq(%w[ce-to-1 ce-to-ee-2 ce-to-ee-231]) - end - end - end - end - end -end diff --git a/scripts/static-analysis b/scripts/static-analysis index 6fd64fbf9da..602cd847a71 100755 --- a/scripts/static-analysis +++ b/scripts/static-analysis @@ -37,10 +37,6 @@ tasks = [ %w[scripts/lint-rugged] ] -if Gitlab.ee? - tasks.unshift(%w[ruby -rbundler/setup scripts/ee_specific_check/ee_specific_check.rb]) -end - static_analysis = Gitlab::Popen::Runner.new static_analysis.run(tasks) do |cmd, &run| |