diff options
Diffstat (limited to 'qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb')
| -rw-r--r-- | qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb | 495 |
1 files changed, 0 insertions, 495 deletions
diff --git a/qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb b/qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb deleted file mode 100644 index e6b60a5b090..00000000000 --- a/qa/qa/specs/features/api/1_manage/import_large_github_repo_spec.rb +++ /dev/null @@ -1,495 +0,0 @@ -# frozen_string_literal: true - -# Lifesize project import test executed from https://gitlab.com/gitlab-org/manage/import/import-metrics - -# rubocop:disable Rails/Pluck -module QA - RSpec.describe 'Manage', :github, requires_admin: 'creates users', only: { job: 'large-github-import' } do - describe 'Project import' do - let(:logger) { Runtime::Logger.logger } - let(:differ) { RSpec::Support::Differ.new(color: true) } - let(:gitlab_address) { QA::Runtime::Scenario.gitlab_address } - let(:dummy_url) { "https://example.com" } - - let(:created_by_pattern) { /\*Created by: \S+\*\n\n/ } - let(:suggestion_pattern) { /suggestion:-\d+\+\d+/ } - let(:gh_link_pattern) { %r{https://github.com/#{github_repo}/(issues|pull)} } - let(:gl_link_pattern) { %r{#{gitlab_address}/#{imported_project.path_with_namespace}/-/(issues|merge_requests)} } - let(:event_pattern) { %r{(un)?assigned( to)? @\S+|mentioned in (issue|merge request) [!#]\d+|changed title from \*\*.*\*\* to \*\*.*\*\*} } # rubocop:disable Layout/LineLength - - let(:api_client) { Runtime::API::Client.as_admin } - - let(:user) do - Resource::User.fabricate_via_api! do |resource| - resource.api_client = api_client - end - end - - let(:github_repo) { ENV['QA_LARGE_IMPORT_REPO'] || 'rspec/rspec-core' } - let(:import_max_duration) { ENV['QA_LARGE_IMPORT_DURATION'] ? ENV['QA_LARGE_IMPORT_DURATION'].to_i : 7200 } - let(:github_client) do - Octokit::Client.new( - access_token: ENV['QA_LARGE_IMPORT_GH_TOKEN'] || Runtime::Env.github_access_token, - auto_paginate: true - ) - end - - let(:gh_repo) { github_client.repository(github_repo) } - - let(:gh_branches) do - logger.debug("= Fetching branches =") - github_client.branches(github_repo).map(&:name) - end - - let(:gh_commits) do - logger.debug("= Fetching commits =") - github_client.commits(github_repo).map(&:sha) - end - - let(:gh_labels) do - logger.debug("= Fetching labels =") - github_client.labels(github_repo).map { |label| { name: label.name, color: "##{label.color}" } } - end - - let(:gh_milestones) do - logger.debug("= Fetching milestones =") - github_client - .list_milestones(github_repo, state: 'all') - .map { |ms| { title: ms.title, description: ms.description } } - end - - let(:gh_all_issues) do - logger.debug("= Fetching issues and prs =") - github_client.list_issues(github_repo, state: 'all') - end - - let(:gh_prs) do - gh_all_issues.select(&:pull_request).each_with_object({}) do |pr, hash| - hash[pr.number] = { - url: pr.html_url, - title: pr.title, - body: pr.body || '', - comments: [*gh_pr_comments[pr.html_url], *gh_issue_comments[pr.html_url]].compact - } - end - end - - let(:gh_issues) do - gh_all_issues.reject(&:pull_request).each_with_object({}) do |issue, hash| - hash[issue.number] = { - url: issue.html_url, - title: issue.title, - body: issue.body || '', - comments: gh_issue_comments[issue.html_url] - } - end - end - - let(:gh_issue_comments) do - logger.debug("= Fetching issue comments =") - github_client.issues_comments(github_repo).each_with_object(Hash.new { |h, k| h[k] = [] }) do |c, hash| - # use base html url as key - hash[c.html_url.gsub(/\#\S+/, "")] << c.body&.gsub(gh_link_pattern, dummy_url) - end - end - - let(:gh_pr_comments) do - logger.debug("= Fetching pr comments =") - github_client.pull_requests_comments(github_repo).each_with_object(Hash.new { |h, k| h[k] = [] }) do |c, hash| - # use base html url as key - hash[c.html_url.gsub(/\#\S+/, "")] << c.body - # some suggestions can contain extra whitespaces which gitlab will remove - &.gsub(/suggestion\s+\r/, "suggestion\r") - &.gsub(gh_link_pattern, dummy_url) - end - end - - let(:imported_project) do - Resource::ProjectImportedFromGithub.fabricate_via_api! do |project| - project.add_name_uuid = false - project.name = 'imported-project' - project.github_personal_access_token = Runtime::Env.github_access_token - project.github_repository_path = github_repo - project.personal_namespace = user.username - project.api_client = api_client - end - end - - # rubocop:disable RSpec/InstanceVariable - after do |example| - next unless defined?(@import_time) - - # save data for comparison notification creation - save_json( - "data", - { - importer: :github, - import_time: @import_time, - errors: imported_project.project_import_status[:failed_relations], - reported_stats: @stats, - source: { - name: "GitHub", - project_name: github_repo, - address: "https://github.com", - data: { - branches: gh_branches.length, - commits: gh_commits.length, - labels: gh_labels.length, - milestones: gh_milestones.length, - mrs: gh_prs.length, - mr_comments: gh_prs.sum { |_k, v| v[:comments].length }, - issues: gh_issues.length, - issue_comments: gh_issues.sum { |_k, v| v[:comments].length } - } - }, - target: { - name: "GitLab", - project_name: imported_project.path_with_namespace, - address: gitlab_address, - data: { - branches: gl_branches.length, - commits: gl_commits.length, - labels: gl_labels.length, - milestones: gl_milestones.length, - mrs: mrs.length, - mr_comments: mrs.sum { |_k, v| v[:comments].length }, - issues: gl_issues.length, - issue_comments: gl_issues.sum { |_k, v| v[:comments].length } - } - }, - not_imported: { - mrs: @mr_diff, - issues: @issue_diff - } - } - ) - end - # rubocop:enable RSpec/InstanceVariable - - it( - 'imports large Github repo via api', - testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347668' - ) do - start = Time.now - - # import the project and log gitlab path - logger.info("== Importing project '#{github_repo}' in to '#{imported_project.reload!.full_path}' ==") - # fetch all objects right after import has started - fetch_github_objects - - import_status = lambda do - imported_project.project_import_status.yield_self do |status| - @stats = status.dig(:stats, :imported) - - # fail fast if import explicitly failed - raise "Import of '#{imported_project.name}' failed!" if status[:import_status] == 'failed' - - status[:import_status] - end - end - - logger.info("== Waiting for import to be finished ==") - expect(import_status).to eventually_eq('finished').within(max_duration: import_max_duration, sleep_interval: 30) - - @import_time = Time.now - start - - aggregate_failures do - verify_repository_import - verify_labels_import - verify_milestones_import - verify_merge_requests_import - verify_issues_import - end - end - - # Persist all objects from repository being imported - # - # @return [void] - def fetch_github_objects - logger.info("== Fetching github repo objects ==") - - gh_repo - gh_branches - gh_commits - gh_labels - gh_milestones - gh_prs - gh_issues - end - - # Verify repository imported correctly - # - # @return [void] - def verify_repository_import - logger.info("== Verifying repository import ==") - expect(imported_project.description).to eq(gh_repo.description) - # check via include, importer creates more branches - # https://gitlab.com/gitlab-org/gitlab/-/issues/332711 - expect(gl_branches).to include(*gh_branches) - expect(gl_commits).to match_array(gh_commits) - end - - # Verify imported labels - # - # @return [void] - def verify_labels_import - logger.info("== Verifying label import ==") - # check via include, additional labels can be inherited from parent group - expect(gl_labels).to include(*gh_labels) - end - - # Verify milestones import - # - # @return [void] - def verify_milestones_import - logger.info("== Verifying milestones import ==") - expect(gl_milestones).to match_array(gh_milestones) - end - - # Verify imported merge requests and mr issues - # - # @return [void] - def verify_merge_requests_import - logger.info("== Verifying merge request import ==") - @mr_diff = verify_mrs_or_issues('mr') - end - - # Verify imported issues and issue comments - # - # @return [void] - def verify_issues_import - logger.info("== Verifying issue import ==") - @issue_diff = verify_mrs_or_issues('issue') - end - - private - - # Verify imported mrs or issues and return missing items - # - # @param [String] type verification object, 'mrs' or 'issues' - # @return [Hash] - def verify_mrs_or_issues(type) - # Compare length to have easy to read overview how many objects are missing - # - expected = type == 'mr' ? mrs : gl_issues - actual = type == 'mr' ? gh_prs : gh_issues - count_msg = "Expected to contain same amount of #{type}s. Gitlab: #{expected.length}, Github: #{actual.length}" - expect(expected.length).to eq(actual.length), count_msg - - missing_comments = verify_comments(type, actual, expected) - - { - "#{type}s": (actual.keys - expected.keys).map { |it| actual[it].slice(:title, :url) }, - "#{type}_comments": missing_comments - } - end - - # Verify imported comments - # - # @param [String] type verification object, 'mrs' or 'issues' - # @param [Hash] actual - # @param [Hash] expected - # @return [Hash] - def verify_comments(type, actual, expected) - actual.each_with_object([]) do |(key, actual_item), missing_comments| - expected_item = expected[key] - title = actual_item[:title] - msg = "expected #{type} with title '#{title}' to have" - - # Print title in the error message to see which object is missing - # - expect(expected_item).to be_truthy, "#{msg} been imported" - next unless expected_item - - # Print difference in the description - # - expected_body = expected_item[:body] - actual_body = actual_item[:body] - body_msg = <<~MSG - #{msg} same description. diff:\n#{differ.diff(expected_body, actual_body)} - MSG - expect(expected_body).to eq(actual_body), body_msg - - # Print amount difference first - # - expected_comments = expected_item[:comments] - actual_comments = actual_item[:comments] - comment_count_msg = <<~MSG - #{msg} same amount of comments. Gitlab: #{expected_comments.length}, Github: #{actual_comments.length} - MSG - expect(expected_comments.length).to eq(actual_comments.length), comment_count_msg - expect(expected_comments).to match_array(actual_comments) - - # Save missing comments - # - comment_diff = actual_comments - expected_comments - next if comment_diff.empty? - - missing_comments << { - title: title, - github_url: actual_item[:url], - gitlab_url: expected_item[:url], - missing_comments: comment_diff - } - end - end - - # Imported project branches - # - # @return [Array] - def gl_branches - @gl_branches ||= begin - logger.debug("= Fetching branches =") - imported_project.repository_branches(auto_paginate: true).map { |b| b[:name] } - end - end - - # Imported project commits - # - # @return [Array] - def gl_commits - @gl_commits ||= begin - logger.debug("= Fetching commits =") - imported_project.commits(auto_paginate: true, attempts: 2).map { |c| c[:id] } - end - end - - # Imported project labels - # - # @return [Array] - def gl_labels - @gl_labels ||= begin - logger.debug("= Fetching labels =") - imported_project.labels(auto_paginate: true).map { |label| label.slice(:name, :color) } - end - end - - # Imported project milestones - # - # @return [<Type>] <description> - def gl_milestones - @gl_milestones ||= begin - logger.debug("= Fetching milestones =") - imported_project.milestones(auto_paginate: true).map { |ms| ms.slice(:title, :description) } - end - end - - # Imported project merge requests - # - # @return [Hash] - def mrs - @mrs ||= begin - logger.debug("= Fetching merge requests =") - imported_mrs = imported_project.merge_requests(auto_paginate: true, attempts: 2) - - logger.debug("= Fetching merge request comments =") - Parallel.map(imported_mrs, in_threads: 4) do |mr| - resource = Resource::MergeRequest.init do |resource| - resource.project = imported_project - resource.iid = mr[:iid] - resource.api_client = api_client - end - - logger.debug("Fetching comments for mr '#{mr[:title]}'") - comments = resource - .comments(auto_paginate: true, attempts: 2) - .reject { |c| c[:system] || c[:body].match?(/^(\*\*Review:\*\*)|(\*Merged by:).*/) } - - [mr[:iid], { - url: mr[:web_url], - title: mr[:title], - body: sanitize_description(mr[:description]) || '', - events: events(comments), - comments: non_event_comments(comments) - }] - end.to_h - end - end - - # Imported project issues - # - # @return [Hash] - def gl_issues - @gl_issues ||= begin - logger.debug("= Fetching issues =") - imported_issues = imported_project.issues(auto_paginate: true, attempts: 2) - - logger.debug("= Fetching issue comments =") - Parallel.map(imported_issues, in_threads: 4) do |issue| - resource = Resource::Issue.init do |issue_resource| - issue_resource.project = imported_project - issue_resource.iid = issue[:iid] - issue_resource.api_client = api_client - end - - logger.debug("Fetching comments for issue '#{issue[:title]}'") - comments = resource.comments(auto_paginate: true, attempts: 2) - - [issue[:iid], { - url: issue[:web_url], - title: issue[:title], - body: sanitize_description(issue[:description]) || '', - events: events(comments), - comments: non_event_comments(comments) - }] - end.to_h - end - end - - # Fetch comments without events - # - # @param [Array] comments - # @return [Array] - def non_event_comments(comments) - comments - .reject { |c| c[:body].match?(event_pattern) } - .map { |c| sanitize_comment(c[:body]) } - end - - # Events - # - # @param [Array] comments - # @return [Array] - def events(comments) - comments - .select { |c| c[:body].match?(event_pattern) } - .map { |c| c[:body] } - end - - # Normalize comments and make them directly comparable - # - # * remove created by prefixes - # * unify suggestion format - # * replace github and gitlab urls - some of the links to objects get transformed to gitlab entities, some don't, - # update all links to example.com for now - # - # @param [String] body - # @return [String] - def sanitize_comment(body) - body - .gsub(created_by_pattern, "") - .gsub(suggestion_pattern, "suggestion\r") - .gsub(gl_link_pattern, dummy_url) - .gsub(gh_link_pattern, dummy_url) - end - - # Remove created by prefix from descripion - # - # @param [String] body - # @return [String] - def sanitize_description(body) - body&.gsub(created_by_pattern, "") - end - - # Save json as file - # - # @param [String] name - # @param [Hash] json - # @return [void] - def save_json(name, json) - File.open("tmp/#{name}.json", "w") { |file| file.write(JSON.pretty_generate(json)) } - end - end - end -end -# rubocop:enable Rails/Pluck |