diff options
-rw-r--r-- | app/controllers/admin/projects_controller.rb | 24 | ||||
-rw-r--r-- | app/mailers/repo_check_mailer.rb | 16 | ||||
-rw-r--r-- | app/views/admin/logs/show.html.haml | 3 | ||||
-rw-r--r-- | app/views/admin/projects/index.html.haml | 15 | ||||
-rw-r--r-- | app/views/admin/projects/show.html.haml | 32 | ||||
-rw-r--r-- | app/views/repo_check_mailer/notify.html.haml | 5 | ||||
-rw-r--r-- | app/views/repo_check_mailer/notify.text.haml | 3 | ||||
-rw-r--r-- | app/workers/admin_email_worker.rb | 12 | ||||
-rw-r--r-- | app/workers/repo_check_worker.rb | 46 | ||||
-rw-r--r-- | app/workers/single_repo_check_worker.rb | 34 | ||||
-rw-r--r-- | config/gitlab.yml.example | 7 | ||||
-rw-r--r-- | config/initializers/1_settings.rb | 7 | ||||
-rw-r--r-- | config/routes.rb | 5 | ||||
-rw-r--r-- | db/migrate/20160315135439_project_add_repo_check.rb | 6 | ||||
-rw-r--r-- | db/schema.rb | 2 | ||||
-rw-r--r-- | doc/administration/repo_checks.md | 39 | ||||
-rw-r--r-- | lib/gitlab/repo_check_logger.rb | 7 | ||||
-rw-r--r-- | spec/features/admin/admin_projects_spec.rb | 37 | ||||
-rw-r--r-- | spec/mailers/repo_check_mailer_spec.rb | 21 | ||||
-rw-r--r-- | spec/workers/repo_check_worker_spec.rb | 31 |
20 files changed, 346 insertions, 6 deletions
diff --git a/app/controllers/admin/projects_controller.rb b/app/controllers/admin/projects_controller.rb index 4089091d569..b8c276fb1bb 100644 --- a/app/controllers/admin/projects_controller.rb +++ b/app/controllers/admin/projects_controller.rb @@ -1,5 +1,5 @@ class Admin::ProjectsController < Admin::ApplicationController - before_action :project, only: [:show, :transfer] + before_action :project, only: [:show, :transfer, :repo_check] before_action :group, only: [:show, :transfer] def index @@ -8,6 +8,7 @@ class Admin::ProjectsController < Admin::ApplicationController @projects = @projects.where("visibility_level IN (?)", params[:visibility_levels]) if params[:visibility_levels].present? @projects = @projects.with_push if params[:with_push].present? @projects = @projects.abandoned if params[:abandoned].present? + @projects = @projects.where(last_repo_check_failed: true) if params[:last_repo_check_failed].present? @projects = @projects.non_archived unless params[:with_archived].present? @projects = @projects.search(params[:name]) if params[:name].present? @projects = @projects.sort(@sort = params[:sort]) @@ -30,6 +31,27 @@ class Admin::ProjectsController < Admin::ApplicationController redirect_to admin_namespace_project_path(@project.namespace, @project) end + def repo_check + SingleRepoCheckWorker.perform_async(@project.id) + + redirect_to( + admin_namespace_project_path(@project.namespace, @project), + notice: 'Repo check was triggered' + ) + end + + def clear_repo_check_states + Project.update_all( + last_repo_check_failed: false, + last_repo_check_at: nil + ) + + redirect_to( + admin_namespaces_projects_path, + notice: 'All project repo check states were cleared' + ) + end + protected def project diff --git a/app/mailers/repo_check_mailer.rb b/app/mailers/repo_check_mailer.rb new file mode 100644 index 00000000000..d98533f120d --- /dev/null +++ b/app/mailers/repo_check_mailer.rb @@ -0,0 +1,16 @@ +class RepoCheckMailer < BaseMailer + include ActionView::Helpers::TextHelper + + def notify(failed_count) + if failed_count == 1 + @message = "One project failed its last repository check" + else + @message = "#{failed_count} projects failed their last repository check" + end + + mail( + to: User.admins.pluck(:email), + subject: @message + ) + end +end diff --git a/app/views/admin/logs/show.html.haml b/app/views/admin/logs/show.html.haml index af9fdeb0734..abcc93f4046 100644 --- a/app/views/admin/logs/show.html.haml +++ b/app/views/admin/logs/show.html.haml @@ -1,6 +1,7 @@ - page_title "Logs" - loggers = [Gitlab::GitLogger, Gitlab::AppLogger, - Gitlab::ProductionLogger, Gitlab::SidekiqLogger] + Gitlab::ProductionLogger, Gitlab::SidekiqLogger, + Gitlab::RepoCheckLogger] %ul.nav-links.log-tabs - loggers.each do |klass| %li{ class: (klass == Gitlab::GitLogger ? 'active' : '') } diff --git a/app/views/admin/projects/index.html.haml b/app/views/admin/projects/index.html.haml index d39c0f44031..ed360f2f012 100644 --- a/app/views/admin/projects/index.html.haml +++ b/app/views/admin/projects/index.html.haml @@ -3,7 +3,7 @@ .row.prepend-top-default %aside.col-md-3 - .admin-filter + .panel.admin-filter = form_tag admin_namespaces_projects_path, method: :get, class: '' do .form-group = label_tag :name, 'Name:' @@ -38,11 +38,22 @@ %span.descr = visibility_level_icon(level) = label - %hr + %fieldset + %strong Problems + .checkbox + = label_tag :last_repo_check_failed do + = check_box_tag :last_repo_check_failed, 1, params[:last_repo_check_failed] + %span Last repo check failed + = hidden_field_tag :sort, params[:sort] = button_tag "Search", class: "btn submit btn-primary" = link_to "Reset", admin_namespaces_projects_path, class: "btn btn-cancel" + .panel.panel-default.repo-check-states + .panel-heading + Repo check states + .panel-body + = link_to 'Clear all', clear_repo_check_states_admin_namespace_projects_path(0), data: { confirm: 'This will clear repo check states for ALL projects in the database. This cannot be undone. Are you sure?' }, method: :put, class: "btn btn-sm btn-remove" %section.col-md-9 .panel.panel-default .panel-heading diff --git a/app/views/admin/projects/show.html.haml b/app/views/admin/projects/show.html.haml index c638c32a654..a7a3f6349ef 100644 --- a/app/views/admin/projects/show.html.haml +++ b/app/views/admin/projects/show.html.haml @@ -5,6 +5,14 @@ %i.fa.fa-pencil-square-o Edit %hr +- if @project.last_repo_check_failed? + .row + .col-md-12 + .panel + .panel-heading.alert.alert-danger + Last repo check failed. See + = link_to 'repocheck.log', admin_logs_path + for error messages. .row .col-md-6 .panel.panel-default @@ -95,6 +103,30 @@ .col-sm-offset-2.col-sm-10 = f.submit 'Transfer', class: 'btn btn-primary' + .panel.panel-default.repo-check + .panel-heading + Repo check + .panel-body + = form_for @project, url: repo_check_admin_namespace_project_path(@project.namespace, @project), method: :post do |f| + .form-group + - if @project.last_repo_check_at.nil? + This repository has never been checked. + - else + This repository was last checked + = @project.last_repo_check_at.to_s(:medium) + '.' + The check + - if @project.last_repo_check_failed? + = succeed '.' do + %strong.cred failed + See + = link_to 'repocheck.log', admin_logs_path + for error messages. + - else + passed. + + .form-group + = f.submit 'Trigger repo check', class: 'btn btn-primary' + .col-md-6 - if @group .panel.panel-default diff --git a/app/views/repo_check_mailer/notify.html.haml b/app/views/repo_check_mailer/notify.html.haml new file mode 100644 index 00000000000..ef0016f9d3e --- /dev/null +++ b/app/views/repo_check_mailer/notify.html.haml @@ -0,0 +1,5 @@ +%p + #{@message}. + +%p + = link_to "See the affected projects in the GitLab admin panel", admin_namespaces_projects_url(last_repo_check_failed: 1) diff --git a/app/views/repo_check_mailer/notify.text.haml b/app/views/repo_check_mailer/notify.text.haml new file mode 100644 index 00000000000..bdf8c2ad675 --- /dev/null +++ b/app/views/repo_check_mailer/notify.text.haml @@ -0,0 +1,3 @@ +#{@message}. +\ +View details: #{admin_namespaces_projects_url(last_repo_check_failed: 1)} diff --git a/app/workers/admin_email_worker.rb b/app/workers/admin_email_worker.rb new file mode 100644 index 00000000000..fcccb9ea669 --- /dev/null +++ b/app/workers/admin_email_worker.rb @@ -0,0 +1,12 @@ +class AdminEmailWorker + include Sidekiq::Worker + + sidekiq_options retry: false # this job auto-repeats via sidekiq-cron + + def perform + repo_check_failed_count = Project.where(last_repo_check_failed: true).count + return if repo_check_failed_count.zero? + + RepoCheckMailer.notify(repo_check_failed_count).deliver_now + end +end diff --git a/app/workers/repo_check_worker.rb b/app/workers/repo_check_worker.rb new file mode 100644 index 00000000000..9be795309e0 --- /dev/null +++ b/app/workers/repo_check_worker.rb @@ -0,0 +1,46 @@ +class RepoCheckWorker + include Sidekiq::Worker + + RUN_TIME = 3600 + + sidekiq_options retry: false + + def perform + start = Time.now + + # This loop will break after a little more than one hour ('a little + # more' because `git fsck` may take a few minutes), or if it runs out of + # projects to check. By default sidekiq-cron will start a new + # RepoCheckWorker each hour so that as long as there are repositories to + # check, only one (or two) will be checked at a time. + project_ids.each do |project_id| + break if Time.now - start >= RUN_TIME + + next if !try_obtain_lease(project_id) + + SingleRepoCheckWorker.new.perform(project_id) + end + end + + private + + # In an ideal world we would use Project.where(...).find_each. + # Unfortunately, calling 'find_each' drops the 'where', so we must build + # an array of IDs instead. + def project_ids + limit = 10_000 + never_checked_projects = Project.where('last_repo_check_at IS NULL').limit(limit). + pluck(:id) + old_check_projects = Project.where('last_repo_check_at < ?', 1.week.ago). + reorder('last_repo_check_at ASC').limit(limit).pluck(:id) + never_checked_projects + old_check_projects + end + + def try_obtain_lease(id) + lease = Gitlab::ExclusiveLease.new( + "project_repo_check:#{id}", + timeout: RUN_TIME + ) + lease.try_obtain + end +end diff --git a/app/workers/single_repo_check_worker.rb b/app/workers/single_repo_check_worker.rb new file mode 100644 index 00000000000..f8b245247c5 --- /dev/null +++ b/app/workers/single_repo_check_worker.rb @@ -0,0 +1,34 @@ +class SingleRepoCheckWorker + include Sidekiq::Worker + + sidekiq_options retry: false + + def perform(project_id) + project = Project.find(project_id) + update(project, success: check(project)) + end + + private + + def check(project) + [project.repository.path_to_repo, project.wiki.wiki.path].all? do |path| + git_fsck(path) + end + end + + def git_fsck(path) + cmd = %W(nice git --git-dir=#{path} fsck) + output, status = Gitlab::Popen.popen(cmd) + return true if status.zero? + + Gitlab::RepoCheckLogger.error("command failed: #{cmd.join(' ')}\n#{output}") + false + end + + def update(project, success:) + project.update_columns( + last_repo_check_failed: !success, + last_repo_check_at: Time.now, + ) + end +end diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index fb1c3476f65..cf20fc9c63a 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -155,6 +155,13 @@ production: &base # Flag stuck CI builds as failed stuck_ci_builds_worker: cron: "0 0 * * *" + # Periodically run 'git fsck' on all repositories. If started more than + # once per hour you will have concurrent 'git fsck' jobs. + repo_check_worker: + cron: "20 * * * *" + # Send admin emails once a day + admin_email_worker: + cron: "0 0 * * *" # diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 2b989015279..8240978ef06 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -239,7 +239,12 @@ Settings['cron_jobs'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_builds_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['stuck_ci_builds_worker']['cron'] ||= '0 0 * * *' Settings.cron_jobs['stuck_ci_builds_worker']['job_class'] = 'StuckCiBuildsWorker' - +Settings.cron_jobs['repo_check_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['repo_check_worker']['cron'] ||= '20 * * * *' +Settings.cron_jobs['repo_check_worker']['job_class'] = 'RepoCheckWorker' +Settings.cron_jobs['admin_email_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['admin_email_worker']['cron'] ||= '0 0 * * *' +Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker' # # GitLab Shell diff --git a/config/routes.rb b/config/routes.rb index 6bf22fb4456..fad8600b77d 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -264,6 +264,11 @@ Rails.application.routes.draw do member do put :transfer + post :repo_check + end + + collection do + put :clear_repo_check_states end resources :runner_projects diff --git a/db/migrate/20160315135439_project_add_repo_check.rb b/db/migrate/20160315135439_project_add_repo_check.rb new file mode 100644 index 00000000000..ba7ddc9ecb4 --- /dev/null +++ b/db/migrate/20160315135439_project_add_repo_check.rb @@ -0,0 +1,6 @@ +class ProjectAddRepoCheck < ActiveRecord::Migration + def change + add_column :projects, :last_repo_check_failed, :boolean, default: false + add_column :projects, :last_repo_check_at, :datetime + end +end diff --git a/db/schema.rb b/db/schema.rb index e63e22ce864..d2c183f968b 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -732,6 +732,8 @@ ActiveRecord::Schema.define(version: 20160331133914) do t.boolean "public_builds", default: true, null: false t.string "main_language" t.integer "pushes_since_gc", default: 0 + t.boolean "last_repo_check_failed", default: false + t.datetime "last_repo_check_at" end add_index "projects", ["builds_enabled", "shared_runners_enabled"], name: "index_projects_on_builds_enabled_and_shared_runners_enabled", using: :btree diff --git a/doc/administration/repo_checks.md b/doc/administration/repo_checks.md new file mode 100644 index 00000000000..9c2c01594e8 --- /dev/null +++ b/doc/administration/repo_checks.md @@ -0,0 +1,39 @@ +# Repo checks + +_**Note:** This feature was [introduced][ce-3232] in GitLab 8.7_ + +--- + +Git has a built-in mechanism [git fsck][git-fsck] to verify the +integrity of all data commited to a repository. GitLab administrators can +trigger such a check for a project via the admin panel. The checks run +asynchronously so it may take a few minutes before the check result is +visible on the project admin page. If the checks failed you can see their +output on the admin log page under 'repocheck.log'. + +## Periodical checks + +GitLab periodically runs a repo check on all project repositories and +wiki repositories in order to detect data corruption problems. A +project will be checked no more than once per week. If any projects +fail their repo checks all GitLab administrators will receive an email +notification of the situation. This notification is sent out no more +than once a day. + + +## What to do if a check failed + +If the repo check fails for some repository you shouldlook up the error +in repocheck.log (in the admin panel or on disk; see +`/var/log/gitlab/gitlab-rails` for Omnibus installations or +`/home/git/gitlab/log` for installations from source). Once you have +resolved the issue use the admin panel to trigger a new repo check on +the project. This will clear the 'check failed' state. + +If for some reason the periodical repo check caused a lot of false +alarms you can choose to clear ALL repo check states from the admin +project index page. + +--- +[ce-3232]: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3232 "Auto git fsck" +[git-fsck]: https://www.kernel.org/pub/software/scm/git/docs/git-fsck.html "git fsck documentation"
\ No newline at end of file diff --git a/lib/gitlab/repo_check_logger.rb b/lib/gitlab/repo_check_logger.rb new file mode 100644 index 00000000000..9409f68722c --- /dev/null +++ b/lib/gitlab/repo_check_logger.rb @@ -0,0 +1,7 @@ +module Gitlab + class RepoCheckLogger < Gitlab::Logger + def self.file_name_noext + 'repocheck' + end + end +end diff --git a/spec/features/admin/admin_projects_spec.rb b/spec/features/admin/admin_projects_spec.rb index 101d955d693..e3991d48ed6 100644 --- a/spec/features/admin/admin_projects_spec.rb +++ b/spec/features/admin/admin_projects_spec.rb @@ -1,6 +1,7 @@ require 'spec_helper' +require 'rails_helper' -describe "Admin::Projects", feature: true do +describe "Admin Projects", feature: true do before do @project = create(:project) login_as :admin @@ -31,4 +32,38 @@ describe "Admin::Projects", feature: true do expect(page).to have_content(@project.name) end end + + feature 'repo checks' do + scenario 'trigger repo check' do + visit_admin_project_page + + page.within('.repo-check') do + click_button 'Trigger repo check' + end + + expect(page).to have_content('Repo check was triggered') + end + + scenario 'see failed repo check' do + @project.update_column(:last_repo_check_failed, true) + visit_admin_project_page + + expect(page).to have_content('Last repo check failed') + end + + scenario 'clear repo checks', js: true do + @project.update_column(:last_repo_check_failed, true) + visit admin_namespaces_projects_path + + page.within('.repo-check-states') do + click_link 'Clear all' # pop-up should be auto confirmed + end + + expect(@project.reload.last_repo_check_failed).to eq(false) + end + end + + def visit_admin_project_page + visit admin_namespace_project_path(@project.namespace, @project) + end end diff --git a/spec/mailers/repo_check_mailer_spec.rb b/spec/mailers/repo_check_mailer_spec.rb new file mode 100644 index 00000000000..d49a6ae0c05 --- /dev/null +++ b/spec/mailers/repo_check_mailer_spec.rb @@ -0,0 +1,21 @@ +require 'rails_helper' + +describe RepoCheckMailer do + include EmailSpec::Matchers + + describe '.notify' do + it 'emails all admins' do + admins = 3.times.map { create(:admin) } + + mail = described_class.notify(1) + + expect(mail).to deliver_to admins.map(&:email) + end + + it 'mentions the number of failed checks' do + mail = described_class.notify(3) + + expect(mail).to have_subject '3 projects failed their last repository check' + end + end +end diff --git a/spec/workers/repo_check_worker_spec.rb b/spec/workers/repo_check_worker_spec.rb new file mode 100644 index 00000000000..7ef3eba9ac5 --- /dev/null +++ b/spec/workers/repo_check_worker_spec.rb @@ -0,0 +1,31 @@ +require 'spec_helper' + +describe RepoCheckWorker do + subject { RepoCheckWorker.new } + + it 'prefers projects that have never been checked' do + projects = 3.times.map { create(:project) } + projects[0].update_column(:last_repo_check_at, 1.month.ago) + projects[2].update_column(:last_repo_check_at, 3.weeks.ago) + + expect(subject.perform).to eq(projects.values_at(1, 0, 2).map(&:id)) + end + + it 'sorts projects by last_repo_check_at' do + projects = 3.times.map { create(:project) } + projects[0].update_column(:last_repo_check_at, 2.weeks.ago) + projects[1].update_column(:last_repo_check_at, 1.month.ago) + projects[2].update_column(:last_repo_check_at, 3.weeks.ago) + + expect(subject.perform).to eq(projects.values_at(1, 2, 0).map(&:id)) + end + + it 'excludes projects that were checked recently' do + projects = 3.times.map { create(:project) } + projects[0].update_column(:last_repo_check_at, 2.days.ago) + projects[1].update_column(:last_repo_check_at, 1.month.ago) + projects[2].update_column(:last_repo_check_at, 3.days.ago) + + expect(subject.perform).to eq([projects[1].id]) + end +end |