diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-15 18:09:53 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-08-15 18:09:53 +0300 |
commit | 51d5328e82229d69456d3a43dd2cf10518bf64c7 (patch) | |
tree | cca8c7aaea10c92398e03ae182b76c5cc514185c /lib | |
parent | 14771dc276c9cfdeed1a3915ee29301cd848b475 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/helpers/integrations_helpers.rb | 14 | ||||
-rw-r--r-- | lib/gitlab/jira_import/issues_importer.rb | 29 |
2 files changed, 35 insertions, 8 deletions
diff --git a/lib/api/helpers/integrations_helpers.rb b/lib/api/helpers/integrations_helpers.rb index 6c6cba43155..99273e81730 100644 --- a/lib/api/helpers/integrations_helpers.rb +++ b/lib/api/helpers/integrations_helpers.rb @@ -328,14 +328,12 @@ module API type: String, desc: '(Advanced) The full URL for your Datadog site' }, - # TODO: uncomment this field once :datadog_integration_logs_collection is rolled out - # https://gitlab.com/gitlab-org/gitlab/-/issues/346339 - # { - # required: false, - # name: :archive_trace_events, - # type: Boolean, - # desc: 'When enabled, job logs will be collected by Datadog and shown along pipeline execution traces' - # }, + { + required: false, + name: :archive_trace_events, + type: Boolean, + desc: 'When enabled, job logs will be collected by Datadog and shown along pipeline execution traces' + }, { required: false, name: :datadog_service, diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index f1ead57c911..25dffcbe0ee 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -7,6 +7,10 @@ module Gitlab # see https://jira.atlassian.com/browse/JRACLOUD-67570 # We set it to 1000 in case they change their mind. BATCH_SIZE = 1000 + JIRA_IMPORT_THRESHOLD = 100_000 + JIRA_IMPORT_PAUSE_LIMIT = 50_000 + + RetriesExceededError = Class.new(RuntimeError) attr_reader :imported_items_cache_key, :start_at, :job_waiter @@ -66,6 +70,10 @@ module Gitlab @issue_type_id, { iid: next_iid } ).execute + + # Pause the importer to allow the import to catch up and cache to drain + pause_jira_issue_importer if jira_import_issue_worker.queue_size > JIRA_IMPORT_THRESHOLD + Gitlab::JiraImport::ImportIssueWorker.perform_async(project.id, jira_issue.id, issue_attrs, job_waiter.key) job_waiter.jobs_remaining += 1 @@ -89,6 +97,27 @@ module Gitlab job_waiter end + def jira_import_issue_worker + @_jira_import_issue_worker ||= Gitlab::JiraImport::ImportIssueWorker + end + + def pause_jira_issue_importer + # Wait for import workers to drop below 50K in the iterations of the timeout + # timeout - Set to 5 seconds. + # Time to process 100K jobs is currently ~14 seconds. + # Source: https://github.com/mperham/sidekiq#performance + # retries - Set to 10 times to avoid indefinitely pause. + # Raises an error if the queue does not reduce below the limit after 10 tries. + + retries = 10 + while retries > 0 && jira_import_issue_worker.queue_size >= JIRA_IMPORT_PAUSE_LIMIT + job_waiter.wait(5) + retries -= 1 + end + + raise RetriesExceededError, 'Retry failed after 10 attempts' if retries == 0 + end + def fetch_issues(start_at) client.Issue.jql("PROJECT='#{jira_project_key}' ORDER BY created ASC", { max_results: BATCH_SIZE, start_at: start_at }) end |