blob: 43402ecd165ae03eb63a1995488fbc575c4033b0 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
|
# frozen_string_literal: true
# This module is used in:
# - SingleEndpointDiffNotesImporter
# - SingleEndpointIssueNotesImporter
# - SingleEndpointMergeRequestNotesImporter
#
# `github_importer_single_endpoint_notes_import`
# feature flag is on.
#
# It fetches 1 PR's associated objects at a time using `issue_comments` or
# `pull_request_comments` endpoint, which is slower than `NotesImporter`
# but it makes sure all notes are imported, as it can sometimes not be
# the case for `NotesImporter`, because `issues_comments` endpoint
# it uses can be limited by GitHub API to not return all available pages.
module Gitlab
module GithubImport
module SingleEndpointNotesImporting
BATCH_SIZE = 100
def each_object_to_import
each_notes_page do |page|
page.objects.each do |note|
next if already_imported?(note)
Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched)
yield(note)
mark_as_imported(note)
end
end
end
def id_for_already_imported_cache(note)
note.id
end
private
def each_notes_page
noteables.each_batch(of: BATCH_SIZE, column: :iid) do |batch|
batch.each do |noteable|
# The page counter needs to be scoped by noteable to avoid skipping
# pages of notes from already imported noteables.
page_counter = PageCounter.new(project, page_counter_id(noteable))
repo = project.import_source
options = collection_options.merge(page: page_counter.current)
client.each_page(collection_method, repo, noteable.iid, options) do |page|
next unless page_counter.set(page.number)
yield page
end
mark_notes_imported(noteable)
end
end
end
def mark_notes_imported(noteable)
Gitlab::Cache::Import::Caching.set_add(
notes_imported_cache_key,
noteable.iid
)
end
def already_imported_noteables
Gitlab::Cache::Import::Caching.values_from_set(notes_imported_cache_key)
end
def noteables
NotImplementedError
end
def notes_imported_cache_key
NotImplementedError
end
def page_counter_id(noteable)
NotImplementedError
end
end
end
end
|
