diff options
Diffstat (limited to 'app/workers')
| -rw-r--r-- | app/workers/admin_email_worker.rb | 6 | ||||
| -rw-r--r-- | app/workers/all_queues.yml | 3 | ||||
| -rw-r--r-- | app/workers/ci/build_trace_chunk_flush_worker.rb | 12 | ||||
| -rw-r--r-- | app/workers/gitlab/github_import/advance_stage_worker.rb | 9 | ||||
| -rw-r--r-- | app/workers/gitlab/github_import/refresh_import_jid_worker.rb | 5 | ||||
| -rw-r--r-- | app/workers/new_note_worker.rb | 2 | ||||
| -rw-r--r-- | app/workers/repository_check/batch_worker.rb | 35 | ||||
| -rw-r--r-- | app/workers/repository_check/single_repository_worker.rb | 48 | ||||
| -rw-r--r-- | app/workers/repository_remove_remote_worker.rb | 35 | ||||
| -rw-r--r-- | app/workers/repository_update_remote_mirror_worker.rb | 49 | ||||
| -rw-r--r-- | app/workers/stuck_import_jobs_worker.rb | 9 |
11 files changed, 169 insertions, 44 deletions
diff --git a/app/workers/admin_email_worker.rb b/app/workers/admin_email_worker.rb index bec0a003a1c..044e470141e 100644 --- a/app/workers/admin_email_worker.rb +++ b/app/workers/admin_email_worker.rb @@ -3,6 +3,12 @@ class AdminEmailWorker include CronjobQueue def perform + send_repository_check_mail if Gitlab::CurrentSettings.repository_checks_enabled + end + + private + + def send_repository_check_mail repository_check_failed_count = Project.where(last_repository_check_failed: true).count return if repository_check_failed_count.zero? diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index c469aea7052..b6433eb3eff 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -52,6 +52,7 @@ - pipeline_creation:create_pipeline - pipeline_creation:run_pipeline_schedule - pipeline_background:archive_trace +- pipeline_background:ci_build_trace_chunk_flush - pipeline_default:build_coverage - pipeline_default:build_trace_sections - pipeline_default:pipeline_metrics @@ -106,9 +107,11 @@ - rebase - repository_fork - repository_import +- repository_remove_remote - storage_migrator - system_hook_push - update_merge_requests - update_user_activity - upload_checksum - web_hook +- repository_update_remote_mirror diff --git a/app/workers/ci/build_trace_chunk_flush_worker.rb b/app/workers/ci/build_trace_chunk_flush_worker.rb new file mode 100644 index 00000000000..218d6688bd9 --- /dev/null +++ b/app/workers/ci/build_trace_chunk_flush_worker.rb @@ -0,0 +1,12 @@ +module Ci + class BuildTraceChunkFlushWorker + include ApplicationWorker + include PipelineBackgroundQueue + + def perform(build_trace_chunk_id) + ::Ci::BuildTraceChunk.find_by(id: build_trace_chunk_id).try do |build_trace_chunk| + build_trace_chunk.use_database! + end + end + end +end diff --git a/app/workers/gitlab/github_import/advance_stage_worker.rb b/app/workers/gitlab/github_import/advance_stage_worker.rb index f7f498af840..8d708e15a66 100644 --- a/app/workers/gitlab/github_import/advance_stage_worker.rb +++ b/app/workers/gitlab/github_import/advance_stage_worker.rb @@ -63,11 +63,10 @@ module Gitlab end def find_project(id) - # We only care about the import JID so we can refresh it. We also only - # want the project if it hasn't been marked as failed yet. It's possible - # the import gets marked as stuck when jobs of the current stage failed - # somehow. - Project.select(:import_jid).import_started.find_by(id: id) + # TODO: Only select the JID + # This is due to the fact that the JID could be present in either the project record or + # its associated import_state record + Project.import_started.find_by(id: id) end end end diff --git a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb index 7108b531bc2..68d2c5c4331 100644 --- a/app/workers/gitlab/github_import/refresh_import_jid_worker.rb +++ b/app/workers/gitlab/github_import/refresh_import_jid_worker.rb @@ -31,7 +31,10 @@ module Gitlab end def find_project(id) - Project.select(:import_jid).import_started.find_by(id: id) + # TODO: Only select the JID + # This is due to the fact that the JID could be present in either the project record or + # its associated import_state record + Project.import_started.find_by(id: id) end end end diff --git a/app/workers/new_note_worker.rb b/app/workers/new_note_worker.rb index b925741934a..67c54fbf10e 100644 --- a/app/workers/new_note_worker.rb +++ b/app/workers/new_note_worker.rb @@ -5,7 +5,7 @@ class NewNoteWorker # old `NewNoteWorker` jobs (can remove later) def perform(note_id, _params = {}) if note = Note.find_by(id: note_id) - NotificationService.new.new_note(note) if note.can_create_notification? + NotificationService.new.new_note(note) Notes::PostProcessService.new(note).execute else Rails.logger.error("NewNoteWorker: couldn't find note with ID=#{note_id}, skipping job") diff --git a/app/workers/repository_check/batch_worker.rb b/app/workers/repository_check/batch_worker.rb index 76688cf51c1..72f0a9b0619 100644 --- a/app/workers/repository_check/batch_worker.rb +++ b/app/workers/repository_check/batch_worker.rb @@ -4,8 +4,11 @@ module RepositoryCheck include CronjobQueue RUN_TIME = 3600 + BATCH_SIZE = 10_000 def perform + return unless Gitlab::CurrentSettings.repository_checks_enabled + start = Time.now # This loop will break after a little more than one hour ('a little @@ -15,7 +18,6 @@ module RepositoryCheck # check, only one (or two) will be checked at a time. project_ids.each do |project_id| break if Time.now - start >= RUN_TIME - break unless current_settings.repository_checks_enabled next unless try_obtain_lease(project_id) @@ -31,12 +33,20 @@ module RepositoryCheck # getting ID's from Postgres is not terribly slow, and because no user # has to sit and wait for this query to finish. def project_ids - limit = 10_000 - never_checked_projects = Project.where('last_repository_check_at IS NULL AND created_at < ?', 24.hours.ago) - .limit(limit).pluck(:id) - old_check_projects = Project.where('last_repository_check_at < ?', 1.month.ago) - .reorder('last_repository_check_at ASC').limit(limit).pluck(:id) - never_checked_projects + old_check_projects + never_checked_project_ids(BATCH_SIZE) + old_checked_project_ids(BATCH_SIZE) + end + + def never_checked_project_ids(batch_size) + Project.where(last_repository_check_at: nil) + .where('created_at < ?', 24.hours.ago) + .limit(batch_size).pluck(:id) + end + + def old_checked_project_ids(batch_size) + Project.where.not(last_repository_check_at: nil) + .where('last_repository_check_at < ?', 1.month.ago) + .reorder(last_repository_check_at: :asc) + .limit(batch_size).pluck(:id) end def try_obtain_lease(id) @@ -47,16 +57,5 @@ module RepositoryCheck timeout: 24.hours ).try_obtain end - - def current_settings - # No caching of the settings! If we cache them and an admin disables - # this feature, an active RepositoryCheckWorker would keep going for up - # to 1 hour after the feature was disabled. - if Rails.env.test? - Gitlab::CurrentSettings.fake_application_settings - else - ApplicationSetting.current - end - end end end diff --git a/app/workers/repository_check/single_repository_worker.rb b/app/workers/repository_check/single_repository_worker.rb index 116bc185b38..3cffb8b14e4 100644 --- a/app/workers/repository_check/single_repository_worker.rb +++ b/app/workers/repository_check/single_repository_worker.rb @@ -5,27 +5,34 @@ module RepositoryCheck def perform(project_id) project = Project.find(project_id) + healthy = project_healthy?(project) + + update_repository_check_status(project, healthy) + end + + private + + def update_repository_check_status(project, healthy) project.update_columns( - last_repository_check_failed: !check(project), + last_repository_check_failed: !healthy, last_repository_check_at: Time.now ) end - private + def project_healthy?(project) + repo_healthy?(project) && wiki_repo_healthy?(project) + end - def check(project) - if has_pushes?(project) && !git_fsck(project.repository) - false - elsif project.wiki_enabled? - # Historically some projects never had their wiki repos initialized; - # this happens on project creation now. Let's initialize an empty repo - # if it is not already there. - project.create_wiki + def repo_healthy?(project) + return true unless has_changes?(project) - git_fsck(project.wiki.repository) - else - true - end + git_fsck(project.repository) + end + + def wiki_repo_healthy?(project) + return true unless has_wiki_changes?(project) + + git_fsck(project.wiki.repository) end def git_fsck(repository) @@ -39,8 +46,19 @@ module RepositoryCheck false end - def has_pushes?(project) + def has_changes?(project) Project.with_push.exists?(project.id) end + + def has_wiki_changes?(project) + return false unless project.wiki_enabled? + + # Historically some projects never had their wiki repos initialized; + # this happens on project creation now. Let's initialize an empty repo + # if it is not already there. + return false unless project.create_wiki + + has_changes?(project) + end end end diff --git a/app/workers/repository_remove_remote_worker.rb b/app/workers/repository_remove_remote_worker.rb new file mode 100644 index 00000000000..1c19b604b77 --- /dev/null +++ b/app/workers/repository_remove_remote_worker.rb @@ -0,0 +1,35 @@ +class RepositoryRemoveRemoteWorker + include ApplicationWorker + include ExclusiveLeaseGuard + + LEASE_TIMEOUT = 1.hour + + attr_reader :project, :remote_name + + def perform(project_id, remote_name) + @remote_name = remote_name + @project = Project.find_by_id(project_id) + + return unless @project + + logger.info("Removing remote #{remote_name} from project #{project.id}") + + try_obtain_lease do + remove_remote = @project.repository.remove_remote(remote_name) + + if remove_remote + logger.info("Remote #{remote_name} was successfully removed from project #{project.id}") + else + logger.error("Could not remove remote #{remote_name} from project #{project.id}") + end + end + end + + def lease_timeout + LEASE_TIMEOUT + end + + def lease_key + "remove_remote_#{project.id}_#{remote_name}" + end +end diff --git a/app/workers/repository_update_remote_mirror_worker.rb b/app/workers/repository_update_remote_mirror_worker.rb new file mode 100644 index 00000000000..bb963979e88 --- /dev/null +++ b/app/workers/repository_update_remote_mirror_worker.rb @@ -0,0 +1,49 @@ +class RepositoryUpdateRemoteMirrorWorker + UpdateAlreadyInProgressError = Class.new(StandardError) + UpdateError = Class.new(StandardError) + + include ApplicationWorker + include Gitlab::ShellAdapter + + sidekiq_options retry: 3, dead: false + + sidekiq_retry_in { |count| 30 * count } + + sidekiq_retries_exhausted do |msg, _| + Sidekiq.logger.warn "Failed #{msg['class']} with #{msg['args']}: #{msg['error_message']}" + end + + def perform(remote_mirror_id, scheduled_time) + remote_mirror = RemoteMirror.find(remote_mirror_id) + return if remote_mirror.updated_since?(scheduled_time) + + raise UpdateAlreadyInProgressError if remote_mirror.update_in_progress? + + remote_mirror.update_start + + project = remote_mirror.project + current_user = project.creator + result = Projects::UpdateRemoteMirrorService.new(project, current_user).execute(remote_mirror) + raise UpdateError, result[:message] if result[:status] == :error + + remote_mirror.update_finish + rescue UpdateAlreadyInProgressError + raise + rescue UpdateError => ex + fail_remote_mirror(remote_mirror, ex.message) + raise + rescue => ex + return unless remote_mirror + + fail_remote_mirror(remote_mirror, ex.message) + raise UpdateError, "#{ex.class}: #{ex.message}" + end + + private + + def fail_remote_mirror(remote_mirror, message) + remote_mirror.mark_as_failed(message) + + Rails.logger.error(message) + end +end diff --git a/app/workers/stuck_import_jobs_worker.rb b/app/workers/stuck_import_jobs_worker.rb index fbb14efc525..6fdd7592e74 100644 --- a/app/workers/stuck_import_jobs_worker.rb +++ b/app/workers/stuck_import_jobs_worker.rb @@ -22,7 +22,8 @@ class StuckImportJobsWorker end def mark_projects_with_jid_as_failed! - jids_and_ids = enqueued_projects_with_jid.pluck(:import_jid, :id).to_h + # TODO: Rollback this change to use SQL through #pluck + jids_and_ids = enqueued_projects_with_jid.map { |project| [project.import_jid, project.id] }.to_h # Find the jobs that aren't currently running or that exceeded the threshold. completed_jids = Gitlab::SidekiqStatus.completed_jids(jids_and_ids.keys) @@ -42,15 +43,15 @@ class StuckImportJobsWorker end def enqueued_projects - Project.with_import_status(:scheduled, :started) + Project.joins_import_state.where("(import_state.status = 'scheduled' OR import_state.status = 'started') OR (projects.import_status = 'scheduled' OR projects.import_status = 'started')") end def enqueued_projects_with_jid - enqueued_projects.where.not(import_jid: nil) + enqueued_projects.where.not("import_state.jid IS NULL AND projects.import_jid IS NULL") end def enqueued_projects_without_jid - enqueued_projects.where(import_jid: nil) + enqueued_projects.where("import_state.jid IS NULL AND projects.import_jid IS NULL") end def error_message |