From 1c5876eb7b2deb069d919bd19b51c9f6218e0f41 Mon Sep 17 00:00:00 2001 From: Dmitriy Zaporozhets Date: Mon, 28 Jan 2013 17:22:45 +0200 Subject: Do gitolite calls async. Remove satellite with project remove --- Procfile | 2 +- app/contexts/projects/create_context.rb | 12 +++-------- app/models/project.rb | 16 +++++++++----- app/models/users_project.rb | 2 +- app/observers/project_observer.rb | 1 + app/workers/gitolite_worker.rb | 10 +++++++++ app/workers/post_receive.rb | 15 +++++++------ lib/gitlab/backend/gitolite.rb | 14 +++++++++--- lib/gitlab/backend/gitolite_config.rb | 38 +++++++++++++-------------------- lib/gitlab/popen.rb | 18 ++++++++++++++++ lib/gitlab/satellite/satellite.rb | 12 +++++++++-- lib/tasks/sidekiq.rake | 2 +- 12 files changed, 90 insertions(+), 52 deletions(-) create mode 100644 app/workers/gitolite_worker.rb create mode 100644 lib/gitlab/popen.rb diff --git a/Procfile b/Procfile index 28a97ddafd7..1a4145cc7d3 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ web: bundle exec unicorn_rails -p $PORT -worker: bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,common,default +worker: bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,common,default,gitolite diff --git a/app/contexts/projects/create_context.rb b/app/contexts/projects/create_context.rb index e644d89a356..915bd8be8b0 100644 --- a/app/contexts/projects/create_context.rb +++ b/app/contexts/projects/create_context.rb @@ -32,16 +32,10 @@ module Projects @project.namespace_id = current_user.namespace_id end - Project.transaction do - @project.creator = current_user - @project.save! + @project.creator = current_user - # Add user as project master - @project.users_projects.create!(project_access: UsersProject::MASTER, user: current_user) - - # when project saved no team member exist so - # project repository should be updated after first user add - @project.update_repository + if @project.save + @project.users_projects.create(project_access: UsersProject::MASTER, user: current_user) end @project diff --git a/app/models/project.rb b/app/models/project.rb index cb6986ce43d..dde15927808 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -299,6 +299,9 @@ class Project < ActiveRecord::Base def trigger_post_receive(oldrev, newrev, ref, user) data = post_receive_data(oldrev, newrev, ref, user) + # Create satellite + self.satellite.create unless self.satellite.exists? + # Create push event self.observe_push(data) @@ -313,9 +316,6 @@ class Project < ActiveRecord::Base self.execute_services(data.dup) end - # Create satellite - self.satellite.create unless self.satellite.exists? - # Discover the default branch, but only if it hasn't already been set to # something else if repository && default_branch.nil? @@ -460,11 +460,17 @@ class Project < ActiveRecord::Base end def update_repository - gitolite.update_repository(self) + GitoliteWorker.perform_async( + :update_repository, + self.id + ) end def destroy_repository - gitolite.remove_repository(self) + GitoliteWorker.perform_async( + :remove_repository, + self.path_with_namespace + ) end def repo_exists? diff --git a/app/models/users_project.rb b/app/models/users_project.rb index ca5048ca7d4..3206600413c 100644 --- a/app/models/users_project.rb +++ b/app/models/users_project.rb @@ -129,7 +129,7 @@ class UsersProject < ActiveRecord::Base end def update_repository - gitolite.update_repository(project) + project.update_repository end def project_access_human diff --git a/app/observers/project_observer.rb b/app/observers/project_observer.rb index b1c694569d7..cc454ae3e35 100644 --- a/app/observers/project_observer.rb +++ b/app/observers/project_observer.rb @@ -10,6 +10,7 @@ class ProjectObserver < ActiveRecord::Observer def after_destroy(project) log_info("Project \"#{project.name}\" was removed") + project.satellite.destroy project.destroy_repository end diff --git a/app/workers/gitolite_worker.rb b/app/workers/gitolite_worker.rb new file mode 100644 index 00000000000..d134ea035f4 --- /dev/null +++ b/app/workers/gitolite_worker.rb @@ -0,0 +1,10 @@ +class GitoliteWorker + include Sidekiq::Worker + include Gitolited + + sidekiq_options queue: :gitolite + + def perform(action, arg) + gitolite.send(action, arg) + end +end diff --git a/app/workers/post_receive.rb b/app/workers/post_receive.rb index e74379a65dd..a906f78f7e3 100644 --- a/app/workers/post_receive.rb +++ b/app/workers/post_receive.rb @@ -13,13 +13,14 @@ class PostReceive # Ignore push from non-gitlab users user = if identifier.eql? Gitlab.config.gitolite.admin_key - email = project.repository.commit(newrev).author.email rescue nil - User.find_by_email(email) if email - elsif /^[A-Z0-9._%a-z\-]+@(?:[A-Z0-9a-z\-]+\.)+[A-Za-z]{2,4}$/.match(identifier) - User.find_by_email(identifier) - else - Key.find_by_identifier(identifier).try(:user) - end + email = project.repository.commit(newrev).author.email rescue nil + User.find_by_email(email) if email + elsif /^[A-Z0-9._%a-z\-]+@(?:[A-Z0-9a-z\-]+\.)+[A-Za-z]{2,4}$/.match(identifier) + User.find_by_email(identifier) + else + Key.find_by_identifier(identifier).try(:user) + end + return false unless user project.trigger_post_receive(oldrev, newrev, ref, user) diff --git a/lib/gitlab/backend/gitolite.rb b/lib/gitlab/backend/gitolite.rb index 3b8a2090f73..d026b76f5ef 100644 --- a/lib/gitlab/backend/gitolite.rb +++ b/lib/gitlab/backend/gitolite.rb @@ -22,7 +22,8 @@ module Gitlab end end - def update_repository project + def update_repository project_id + project = Project.find(project_id) config.update_project!(project) end @@ -33,8 +34,15 @@ module Gitlab end end - def remove_repository project - config.destroy_project!(project) + # Remove repository from gitolite + # + # name - project path with namespace + # + # Ex. + # remove_repository("gitlab/gitlab-ci") + # + def remove_repository(name) + config.destroy_project!(name) end def url_to_repo path diff --git a/lib/gitlab/backend/gitolite_config.rb b/lib/gitlab/backend/gitolite_config.rb index e4ebd595a93..748f9d74390 100644 --- a/lib/gitlab/backend/gitolite_config.rb +++ b/lib/gitlab/backend/gitolite_config.rb @@ -4,6 +4,8 @@ require 'fileutils' module Gitlab class GitoliteConfig + include Gitlab::Popen + class PullError < StandardError; end class PushError < StandardError; end class BrokenGitolite < StandardError; end @@ -87,12 +89,14 @@ module Gitlab Gitlab::GitLogger.error(message) end - def destroy_project(project) - # do rm-rf only if repository exists - if project.repository - FileUtils.rm_rf(project.repository.path_to_repo) - end - conf.rm_repo(project.path_with_namespace) + def path_to_repo(name) + File.join(Gitlab.config.gitolite.repos_path, "#{name}.git") + end + + def destroy_project(name) + full_path = path_to_repo(name) + FileUtils.rm_rf(full_path) if File.exists?(full_path) + conf.rm_repo(name) end def clean_repo repo_name @@ -210,14 +214,14 @@ module Gitlab end def push - output, status = popen('git add -A') + output, status = popen('git add -A', tmp_conf_path) raise "Git add failed." unless status.zero? # git commit returns 0 on success, and 1 if there is nothing to commit - output, status = popen('git commit -m "GitLab"') + output, status = popen('git commit -m "GitLab"', tmp_conf_path) raise "Git add failed." unless [0,1].include?(status) - output, status = popen('git push') + output, status = popen('git push', tmp_conf_path) if output =~ /remote\: FATAL/ raise BrokenGitolite, output @@ -230,20 +234,8 @@ module Gitlab end end - def popen(cmd, path = nil) - path ||= File.join(config_tmp_dir,'gitolite') - vars = { "PWD" => path } - options = { :chdir => path } - - @cmd_output = "" - @cmd_status = 0 - Open3.popen3(vars, cmd, options) do |stdin, stdout, stderr, wait_thr| - @cmd_status = wait_thr.value.exitstatus - @cmd_output << stdout.read - @cmd_output << stderr.read - end - - return @cmd_output, @cmd_status + def tmp_conf_path + File.join(config_tmp_dir,'gitolite') end end end diff --git a/lib/gitlab/popen.rb b/lib/gitlab/popen.rb new file mode 100644 index 00000000000..f2cfd8073e3 --- /dev/null +++ b/lib/gitlab/popen.rb @@ -0,0 +1,18 @@ +module Gitlab + module Popen + def popen(cmd, path) + vars = { "PWD" => path } + options = { :chdir => path } + + @cmd_output = "" + @cmd_status = 0 + Open3.popen3(vars, cmd, options) do |stdin, stdout, stderr, wait_thr| + @cmd_status = wait_thr.value.exitstatus + @cmd_output << stdout.read + @cmd_output << stderr.read + end + + return @cmd_output, @cmd_status + end + end +end diff --git a/lib/gitlab/satellite/satellite.rb b/lib/gitlab/satellite/satellite.rb index 164af55d895..d8e8f58963b 100644 --- a/lib/gitlab/satellite/satellite.rb +++ b/lib/gitlab/satellite/satellite.rb @@ -3,6 +3,8 @@ module Gitlab module Satellite class Satellite + include Gitlab::Popen + PARKING_BRANCH = "__parking_branch" attr_accessor :project @@ -24,8 +26,10 @@ module Gitlab end def create - create_cmd = "git clone #{project.url_to_repo} #{path}" - if system(create_cmd) + output, status = popen("git clone #{project.url_to_repo} #{path}", + Gitlab.config.satellites.path) + + if status.zero? true else Gitlab::GitLogger.error("Failed to create satellite for #{project.name_with_namespace}") @@ -66,6 +70,10 @@ module Gitlab @repo ||= Grit::Repo.new(path) end + def destroy + FileUtils.rm_rf(path) + end + private # Clear the working directory diff --git a/lib/tasks/sidekiq.rake b/lib/tasks/sidekiq.rake index 0d2ec6f332c..e4eb0e6776f 100644 --- a/lib/tasks/sidekiq.rake +++ b/lib/tasks/sidekiq.rake @@ -6,7 +6,7 @@ namespace :sidekiq do desc "GITLAB | Start sidekiq" task :start do - run "nohup bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,common,default -e #{Rails.env} -P #{pidfile} >> #{Rails.root.join("log", "sidekiq.log")} 2>&1 &" + run "nohup bundle exec sidekiq -q post_receive,mailer,system_hook,project_web_hook,gitolite,common,default -e #{Rails.env} -P #{pidfile} >> #{Rails.root.join("log", "sidekiq.log")} 2>&1 &" end def pidfile -- cgit v1.2.3