diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-19 11:27:35 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-19 11:27:35 +0300 |
commit | 7e9c479f7de77702622631cff2628a9c8dcbc627 (patch) | |
tree | c8f718a08e110ad7e1894510980d2155a6549197 /lib/gitlab/github_import | |
parent | e852b0ae16db4052c1c567d9efa4facc81146e88 (diff) |
Add latest changes from gitlab-org/gitlab@13-6-stable-eev13.6.0-rc42
Diffstat (limited to 'lib/gitlab/github_import')
-rw-r--r-- | lib/gitlab/github_import/client.rb | 53 | ||||
-rw-r--r-- | lib/gitlab/github_import/sequential_importer.rb | 5 |
2 files changed, 53 insertions, 5 deletions
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 22803c5cd71..dfe60fb5a03 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -18,6 +18,8 @@ module Gitlab attr_reader :octokit + SEARCH_MAX_REQUESTS_PER_MINUTE = 30 + # A single page of data and the corresponding page number. Page = Struct.new(:objects, :number) @@ -28,9 +30,12 @@ module Gitlab # rate limit at once. The threshold is put in place to not hit the limit # in most cases. RATE_LIMIT_THRESHOLD = 50 + SEARCH_RATE_LIMIT_THRESHOLD = 3 # token - The GitHub API token to use. # + # host - The GitHub hostname. If nil, github.com will be used. + # # per_page - The number of objects that should be displayed per page. # # parallel - When set to true hitting the rate limit will result in a @@ -39,11 +44,13 @@ module Gitlab # this value to `true` for parallel importing is crucial as # otherwise hitting the rate limit will result in a thread # being blocked in a `sleep()` call for up to an hour. - def initialize(token, per_page: 100, parallel: true) + def initialize(token, host: nil, per_page: 100, parallel: true) + @host = host @octokit = ::Octokit::Client.new( access_token: token, per_page: per_page, - api_endpoint: api_endpoint + api_endpoint: api_endpoint, + web_endpoint: web_endpoint ) @octokit.connection_options[:ssl] = { verify: verify_ssl } @@ -148,8 +155,26 @@ module Gitlab end end + def search_repos_by_name(name) + each_page(:search_repositories, search_query(str: name, type: :name)) + end + + def search_query(str:, type:, include_collaborations: true, include_orgs: true) + query = "#{str} in:#{type} is:public,private user:#{octokit.user.login}" + + query = [query, collaborations_subquery].join(' ') if include_collaborations + query = [query, organizations_subquery].join(' ') if include_orgs + + query + end + # Returns `true` if we're still allowed to perform API calls. + # Search API has rate limit of 30, use lowered threshold when search is used. def requests_remaining? + if requests_limit == SEARCH_MAX_REQUESTS_PER_MINUTE + return remaining_requests > SEARCH_RATE_LIMIT_THRESHOLD + end + remaining_requests > RATE_LIMIT_THRESHOLD end @@ -157,6 +182,10 @@ module Gitlab octokit.rate_limit.remaining end + def requests_limit + octokit.rate_limit.limit + end + def raise_or_wait_for_rate_limit rate_limit_counter.increment @@ -181,7 +210,11 @@ module Gitlab end def api_endpoint - custom_api_endpoint || default_api_endpoint + @host || custom_api_endpoint || default_api_endpoint + end + + def web_endpoint + @host || custom_api_endpoint || ::Octokit::Default.web_endpoint end def custom_api_endpoint @@ -213,6 +246,20 @@ module Gitlab 'The number of GitHub API calls performed when importing projects' ) end + + private + + def collaborations_subquery + each_object(:repos, nil, { affiliation: 'collaborator' }) + .map { |repo| "repo:#{repo.full_name}" } + .join(' ') + end + + def organizations_subquery + each_object(:organizations) + .map { |org| "org:#{org.login}" } + .join(' ') + end end end end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb index 6a181caf65d..cb6b2017208 100644 --- a/lib/gitlab/github_import/sequential_importer.rb +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -25,10 +25,11 @@ module Gitlab # project - The project to import the data into. # token - The token to use for the GitHub API. - def initialize(project, token: nil) + # host - The GitHub hostname. If nil, github.com will be used. + def initialize(project, token: nil, host: nil) @project = project @client = GithubImport - .new_client_for(project, token: token, parallel: false) + .new_client_for(project, token: token, host: host, parallel: false) end def execute |