diff options
author | James Lopez <james@jameslopez.es> | 2016-06-30 15:37:48 +0300 |
---|---|---|
committer | James Lopez <james@jameslopez.es> | 2016-06-30 15:37:48 +0300 |
commit | 67f59ebd50511bfba1ebdc73b6fd269f425ac6f0 (patch) | |
tree | 635c708678523ca5149880c1bea6dd8fa73c0a0f /lib | |
parent | 545b92af067832d560846b7ec7cb26caf3302275 (diff) | |
parent | 1a68a0a6f55049b4ad42222b0dc667fe625a7604 (diff) |
Merge branch 'master' of gitlab.com:gitlab-org/gitlab-ce into fix/import-url-validator
# Conflicts:
# spec/models/project_spec.rb
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/builds.rb | 6 | ||||
-rw-r--r-- | lib/api/entities.rb | 1 | ||||
-rw-r--r-- | lib/api/group_members.rb | 2 | ||||
-rw-r--r-- | lib/api/internal.rb | 36 | ||||
-rw-r--r-- | lib/backup/repository.rb | 26 | ||||
-rw-r--r-- | lib/gitlab.rb | 7 | ||||
-rw-r--r-- | lib/gitlab/backend/grack_auth.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/backend/shell.rb | 98 | ||||
-rw-r--r-- | lib/gitlab/current_settings.rb | 10 | ||||
-rw-r--r-- | lib/gitlab/github_import/importer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/gon_helper.rb | 1 | ||||
-rw-r--r-- | lib/gitlab/metrics/method_call.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/metrics/metric.rb | 21 | ||||
-rw-r--r-- | lib/gitlab/metrics/system.rb | 20 | ||||
-rw-r--r-- | lib/gitlab/metrics/transaction.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/o_auth/auth_hash.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/workhorse.rb | 13 | ||||
-rw-r--r-- | lib/tasks/gitlab/check.rake | 167 | ||||
-rw-r--r-- | lib/tasks/gitlab/cleanup.rake | 71 | ||||
-rw-r--r-- | lib/tasks/gitlab/import.rake | 96 | ||||
-rw-r--r-- | lib/tasks/gitlab/info.rake | 5 | ||||
-rw-r--r-- | lib/tasks/gitlab/list_repos.rake | 2 | ||||
-rw-r--r-- | lib/tasks/gitlab/shell.rake | 12 | ||||
-rw-r--r-- | lib/tasks/gitlab/task_helpers.rake | 12 |
24 files changed, 358 insertions, 266 deletions
diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 979328efe0e..086d8511e8f 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -33,10 +33,10 @@ module API get ':id/repository/commits/:sha/builds' do authorize_read_builds! - commit = user_project.pipelines.find_by_sha(params[:sha]) - return not_found! unless commit + return not_found! unless user_project.commit(params[:sha]) - builds = commit.builds.order('id DESC') + pipelines = user_project.pipelines.where(sha: params[:sha]) + builds = user_project.builds.where(pipeline: pipelines).order('id DESC') builds = filter_builds(builds, params[:scope]) present paginate(builds), with: Entities::Build, diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 5a23a18fe9c..4e2a43e45e2 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -376,6 +376,7 @@ module API expose :user_oauth_applications expose :after_sign_out_path expose :container_registry_token_expire_delay + expose :repository_storage end class Release < Grape::Entity diff --git a/lib/api/group_members.rb b/lib/api/group_members.rb index ab9b7c602b5..dbe5bb08d3f 100644 --- a/lib/api/group_members.rb +++ b/lib/api/group_members.rb @@ -77,7 +77,7 @@ module API member = group.group_members.find_by(user_id: params[:user_id]) if member.nil? - render_api_error!("404 Not Found - user_id:#{params[:user_id]} not a member of group #{group.name}",404) + render_api_error!("404 Not Found - user_id:#{params[:user_id]} not a member of group #{group.name}", 404) else member.destroy end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 1d361569d59..b32503e8516 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -20,6 +20,20 @@ module API @wiki ||= params[:project].end_with?('.wiki') && !Project.find_with_namespace(params[:project]) end + + def project + @project ||= begin + project_path = params[:project] + + # Check for *.wiki repositories. + # Strip out the .wiki from the pathname before finding the + # project. This applies the correct project permissions to + # the wiki repository as well. + project_path.chomp!('.wiki') if wiki? + + Project.find_with_namespace(project_path) + end + end end post "/allowed" do @@ -32,16 +46,6 @@ module API User.find_by(id: params[:user_id]) end - project_path = params[:project] - - # Check for *.wiki repositories. - # Strip out the .wiki from the pathname before finding the - # project. This applies the correct project permissions to - # the wiki repository as well. - project_path.chomp!('.wiki') if wiki? - - project = Project.find_with_namespace(project_path) - access = if wiki? Gitlab::GitAccessWiki.new(actor, project) @@ -49,7 +53,17 @@ module API Gitlab::GitAccess.new(actor, project) end - access.check(params[:action], params[:changes]) + access_status = access.check(params[:action], params[:changes]) + + response = { status: access_status.status, message: access_status.message } + + if access_status.status + # Return the repository full path so that gitlab-shell has it when + # handling ssh commands + response[:repository_path] = project.repository.path_to_repo + end + + response end # diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 7b91215d50b..b9773f98d75 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -2,8 +2,6 @@ require 'yaml' module Backup class Repository - attr_reader :repos_path - def dump prepare @@ -50,10 +48,12 @@ module Backup end def restore - if File.exists?(repos_path) + Gitlab.config.repositories.storages.each do |name, path| + next unless File.exists?(path) + # Move repos dir to 'repositories.old' dir - bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s) - FileUtils.mv(repos_path, bk_repos_path) + bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) + FileUtils.mv(path, bk_repos_path) end FileUtils.mkdir_p(repos_path) @@ -61,7 +61,7 @@ module Backup Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " - project.namespace.ensure_dir_exist if project.namespace + project.ensure_dir_exist if File.exists?(path_to_bundle(project)) FileUtils.mkdir_p(path_to_repo(project)) @@ -100,8 +100,8 @@ module Backup end $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) - cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks" - if system(cmd) + cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args + if system(*cmd) $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) @@ -120,10 +120,6 @@ module Backup File.join(backup_repos_path, project.path_with_namespace + ".bundle") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - def backup_repos_path File.join(Gitlab.config.backup.path, "repositories") end @@ -139,5 +135,11 @@ module Backup def silent {err: '/dev/null', out: '/dev/null'} end + + private + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end end diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 37f4c34054f..c3064163e07 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -2,6 +2,11 @@ require_dependency 'gitlab/git' module Gitlab def self.com? - Gitlab.config.gitlab.url == 'https://gitlab.com' + # Check `staging?` as well to keep parity with gitlab.com + Gitlab.config.gitlab.url == 'https://gitlab.com' || staging? + end + + def self.staging? + Gitlab.config.gitlab.url == 'https://staging.gitlab.com' end end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index ab7b811c5d8..ad412f56cca 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -22,7 +22,7 @@ module Grack # Need this if under RELATIVE_URL_ROOT unless Gitlab.config.gitlab.relative_url_root.empty? # If website is mounted using relative_url_root need to remove it first - @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root,'') + @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root, '') else @env['PATH_INFO'] = @request.path end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 3e3986d6382..34e0143a82e 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -1,3 +1,5 @@ +require 'securerandom' + module Gitlab class Shell class Error < StandardError; end @@ -18,77 +20,82 @@ module Gitlab # Init new repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # add_repository("gitlab/gitlab-ci") + # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - def add_repository(name) + def add_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'add-project', "#{name}.git"]) + 'add-project', storage, "#{name}.git"]) end # Import repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") + # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # - def import_repository(name, url) - output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900']) + def import_repository(storage, name, url) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, '900']) raise Error, output unless status.zero? true end # Move repository - # + # storage - project's storage path # path - project path with namespace # new_path - new project path with namespace # # Ex. - # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new") + # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - def mv_repository(path, new_path) + def mv_repository(storage, path, new_path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', - "#{path}.git", "#{new_path}.git"]) + storage, "#{path}.git", "#{new_path}.git"]) end # Fork repository to new namespace - # + # storage - project's storage path # path - project path with namespace # fork_namespace - namespace for forked project # # Ex. - # fork_repository("gitlab/gitlab-ci", "randx") + # fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx") # - def fork_repository(path, fork_namespace) + def fork_repository(storage, path, fork_namespace) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', - "#{path}.git", fork_namespace]) + storage, "#{path}.git", fork_namespace]) end # Remove repository from file system # + # storage - project's storage path # name - project path with namespace # # Ex. - # remove_repository("gitlab/gitlab-ci") + # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - def remove_repository(name) + def remove_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'rm-project', "#{name}.git"]) + 'rm-project', storage, "#{name}.git"]) end # Gc repository # + # storage - project storage path # path - project path with namespace # # Ex. - # gc("gitlab/gitlab-ci") + # gc("/path/to/storage", "gitlab/gitlab-ci") # - def gc(path) + def gc(storage, path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', - "#{path}.git"]) + storage, "#{path}.git"]) end # Add new key to gitlab-shell @@ -133,31 +140,31 @@ module Gitlab # Add empty directory for storing repositories # # Ex. - # add_namespace("gitlab") + # add_namespace("/path/to/storage", "gitlab") # - def add_namespace(name) - FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name) + def add_namespace(storage, name) + FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name) end # Remove directory from repositories storage # Every repository inside this directory will be removed too # # Ex. - # rm_namespace("gitlab") + # rm_namespace("/path/to/storage", "gitlab") # - def rm_namespace(name) - FileUtils.rm_r(full_path(name), force: true) + def rm_namespace(storage, name) + FileUtils.rm_r(full_path(storage, name), force: true) end # Move namespace directory inside repositories storage # # Ex. - # mv_namespace("gitlab", "gitlabhq") + # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - def mv_namespace(old_name, new_name) - return false if exists?(new_name) || !exists?(old_name) + def mv_namespace(storage, old_name, new_name) + return false if exists?(storage, new_name) || !exists?(storage, old_name) - FileUtils.mv(full_path(old_name), full_path(new_name)) + FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) end def url_to_repo(path) @@ -176,11 +183,26 @@ module Gitlab # Check if such directory exists in repositories. # # Usage: - # exists?('gitlab') - # exists?('gitlab/cookies.git') + # exists?(storage, 'gitlab') + # exists?(storage, 'gitlab/cookies.git') # - def exists?(dir_name) - File.exist?(full_path(dir_name)) + def exists?(storage, dir_name) + File.exist?(full_path(storage, dir_name)) + end + + # Create (if necessary) and link the secret token file + def generate_and_link_secret_token + secret_file = Gitlab.config.gitlab_shell.secret_file + unless File.exist? secret_file + # Generate a new token of 16 random hexadecimal characters and store it in secret_file. + token = SecureRandom.hex(16) + File.write(secret_file, token) + end + + link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret') + if File.exist?(gitlab_shell_path) && !File.exist?(link_path) + FileUtils.symlink(secret_file, link_path) + end end protected @@ -193,14 +215,10 @@ module Gitlab File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - - def full_path(dir_name) + def full_path(storage, dir_name) raise ArgumentError.new("Directory name can't be blank") if dir_name.blank? - File.join(repos_path, dir_name) + File.join(storage, dir_name) end def gitlab_shell_projects_path diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 28c34429c1f..54b46e5d23f 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -9,10 +9,14 @@ module Gitlab end def ensure_application_settings! - settings = ::ApplicationSetting.cached + if connect_to_db? + begin + settings = ::ApplicationSetting.current + # In case Redis isn't running or the Redis UNIX socket file is not available + rescue ::Redis::BaseError, ::Errno::ENOENT + settings = ::ApplicationSetting.last + end - if !settings && connect_to_db? - settings = ::ApplicationSetting.current settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 2286ac8829c..730978d502b 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -167,7 +167,7 @@ module Gitlab def import_wiki unless project.wiki_enabled? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) project.update_attribute(:wiki_enabled, true) end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index f751a3a12fd..d4f12cb1df9 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -3,7 +3,6 @@ module Gitlab def add_gon_variables gon.api_version = API::API.version gon.default_avatar_url = URI::join(Gitlab.config.gitlab.url, ActionController::Base.helpers.image_path('no_avatar.png')).to_s - gon.default_issues_tracker = Project.new.default_issue_tracker.to_param gon.max_file_size = current_application_settings.max_attachment_size gon.relative_url_root = Gitlab.config.gitlab.relative_url_root gon.shortcuts_path = help_shortcuts_path diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index faf0d9b6318..c048fe20ba7 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -18,12 +18,12 @@ module Gitlab # Measures the real and CPU execution time of the supplied block. def measure - start_real = Time.now + start_real = System.monotonic_time start_cpu = System.cpu_time retval = yield - @real_time += (Time.now - start_real) * 1000.0 - @cpu_time += System.cpu_time.to_f - start_cpu + @real_time += System.monotonic_time - start_real + @cpu_time += System.cpu_time - start_cpu @call_count += 1 retval diff --git a/lib/gitlab/metrics/metric.rb b/lib/gitlab/metrics/metric.rb index 1cd1ca30f70..f23d67e1e38 100644 --- a/lib/gitlab/metrics/metric.rb +++ b/lib/gitlab/metrics/metric.rb @@ -4,16 +4,15 @@ module Gitlab class Metric JITTER_RANGE = 0.000001..0.001 - attr_reader :series, :values, :tags, :created_at + attr_reader :series, :values, :tags # series - The name of the series (as a String) to store the metric in. # values - A Hash containing the values to store. # tags - A Hash containing extra tags to add to the metrics. def initialize(series, values, tags = {}) - @values = values - @series = series - @tags = tags - @created_at = Time.now.utc + @values = values + @series = series + @tags = tags end # Returns a Hash in a format that can be directly written to InfluxDB. @@ -27,20 +26,20 @@ module Gitlab # # Due to the way InfluxDB is set up there's no solution to this problem, # all we can do is lower the amount of collisions. We do this by using - # Time#to_f which returns the seconds as a Float providing greater - # accuracy. We then add a small random value that is large enough to - # distinguish most timestamps but small enough to not alter the amount - # of seconds. + # System.real_time which returns the nanoseconds as a Float providing + # greater accuracy. We then add a small random value that is large + # enough to distinguish most timestamps but small enough to not alter + # the timestamp significantly. # # See https://gitlab.com/gitlab-com/operations/issues/175 for more # information. - time = @created_at.to_f + rand(JITTER_RANGE) + time = System.real_time(:nanosecond) + rand(JITTER_RANGE) { series: @series, tags: @tags, values: @values, - timestamp: (time * 1_000_000_000).to_i + timestamp: time.to_i } end end diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index a7d183b2f94..82c18bb108b 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -34,13 +34,29 @@ module Gitlab # THREAD_CPUTIME is not supported on OS X if Process.const_defined?(:CLOCK_THREAD_CPUTIME_ID) def self.cpu_time - Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond) + Process. + clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :millisecond).to_f end else def self.cpu_time - Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond) + Process. + clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :millisecond).to_f end end + + # Returns the current real time in a given precision. + # + # Returns the time as a Float. + def self.real_time(precision = :millisecond) + Process.clock_gettime(Process::CLOCK_REALTIME, precision).to_f + end + + # Returns the current monotonic clock time in a given precision. + # + # Returns the time as a Float. + def self.monotonic_time(precision = :millisecond) + Process.clock_gettime(Process::CLOCK_MONOTONIC, precision).to_f + end end end end diff --git a/lib/gitlab/metrics/transaction.rb b/lib/gitlab/metrics/transaction.rb index 4bc5081aa03..bded245da43 100644 --- a/lib/gitlab/metrics/transaction.rb +++ b/lib/gitlab/metrics/transaction.rb @@ -30,7 +30,7 @@ module Gitlab end def duration - @finished_at ? (@finished_at - @started_at) * 1000.0 : 0.0 + @finished_at ? (@finished_at - @started_at) : 0.0 end def allocated_memory @@ -41,12 +41,12 @@ module Gitlab Thread.current[THREAD_KEY] = self @memory_before = System.memory_usage - @started_at = Time.now + @started_at = System.monotonic_time yield ensure @memory_after = System.memory_usage - @finished_at = Time.now + @finished_at = System.monotonic_time Thread.current[THREAD_KEY] = nil end diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index 36e5c2670bb..7d6911a1ab3 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -66,7 +66,7 @@ module Gitlab # Get the first part of the email address (before @) # In addtion in removes illegal characters def generate_username(email) - email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/,'').to_s + email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s end def generate_temporarily_email(username) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 40e8299c36b..ef1241f8600 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -52,6 +52,19 @@ module Gitlab ] end + def send_git_patch(repository, from, to) + params = { + 'RepoPath' => repository.path_to_repo, + 'ShaFrom' => from, + 'ShaTo' => to + } + + [ + SEND_DATA_HEADER, + "git-format-patch:#{encode(params)}" + ] + end + protected def encode(hash) diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 12d6ac45fb6..e9a4e37ec48 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -356,97 +356,108 @@ namespace :gitlab do ######################## def check_repo_base_exists - print "Repo base directory exists? ... " + puts "Repo base directory exists?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.exists?(repo_base_path) - puts "yes".color(:green) - else - puts "no".color(:red) - puts "#{repo_base_path} is missing".color(:red) - try_fixing_it( - "This should have been created when setting up GitLab Shell.", - "Make sure it's set correctly in config/gitlab.yml", - "Make sure GitLab Shell is installed correctly." - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + if File.exists?(repo_base_path) + puts "yes".color(:green) + else + puts "no".color(:red) + puts "#{repo_base_path} is missing".color(:red) + try_fixing_it( + "This should have been created when setting up GitLab Shell.", + "Make sure it's set correctly in config/gitlab.yml", + "Make sure GitLab Shell is installed correctly." + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_is_not_symlink - print "Repo base directory is a symlink? ... " + puts "Repo storage directories are symlinks?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - unless File.symlink?(repo_base_path) - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Make sure it's set to the real directory in config/gitlab.yml" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + unless File.symlink?(repo_base_path) + puts "no".color(:green) + else + puts "yes".color(:red) + try_fixing_it( + "Make sure it's set to the real directory in config/gitlab.yml" + ) + fix_and_rerun + end end end def check_repo_base_permissions - print "Repo base access is drwxrws---? ... " + puts "Repo paths access is drwxrws---?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", - "sudo chmod -R ug-s #{repo_base_path}", - "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") + puts "yes".color(:green) + else + puts "no".color(:red) + try_fixing_it( + "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", + "sudo chmod -R ug-s #{repo_base_path}", + "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_user_and_group gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group - print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... " + puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - uid = uid_for(gitlab_shell_ssh_user) - gid = gid_for(gitlab_shell_owner_group) - if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid - puts "yes".color(:green) - else - puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) - try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + uid = uid_for(gitlab_shell_ssh_user) + gid = gid_for(gitlab_shell_owner_group) + if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid + puts "yes".color(:green) + else + puts "no".color(:red) + puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) + try_fixing_it( + "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end @@ -473,7 +484,7 @@ namespace :gitlab do else puts "wrong or missing hooks".color(:red) try_fixing_it( - sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"), + sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"), 'Check the hooks_path in config/gitlab.yml', 'Check your gitlab-shell installation' ) @@ -785,13 +796,13 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - namespace_dirs = Dir.glob( - File.join(Gitlab.config.gitlab_shell.repos_path, '*') - ) + Gitlab.config.repositories.storages.each do |name, path| + namespace_dirs = Dir.glob(File.join(path, '*')) - namespace_dirs.each do |namespace_dir| - repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + namespace_dirs.each do |namespace_dir| + repo_dirs = Dir.glob(File.join(namespace_dir, '*')) + repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + end end end end @@ -799,12 +810,12 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| - username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue)) + username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| File.join( - Gitlab.config.gitlab_shell.repos_path, + p.repository_storage_path, "#{p.path_with_namespace}.git" ) end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index ab0028d6603..b7cbdc6cd78 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -5,36 +5,36 @@ namespace :gitlab do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) - git_base_path = Gitlab.config.gitlab_shell.repos_path - all_dirs = Dir.glob(git_base_path + '/*') + Gitlab.config.repositories.storages.each do |name, git_base_path| + all_dirs = Dir.glob(git_base_path + '/*') - puts git_base_path.color(:yellow) - puts "Looking for directories to remove... " + puts git_base_path.color(:yellow) + puts "Looking for directories to remove... " - all_dirs.reject! do |dir| - # skip if git repo - dir =~ /.git$/ - end + all_dirs.reject! do |dir| + # skip if git repo + dir =~ /.git$/ + end - all_dirs.reject! do |dir| - dir_name = File.basename dir + all_dirs.reject! do |dir| + dir_name = File.basename dir - # skip if namespace present - namespaces.include?(dir_name) - end + # skip if namespace present + namespaces.include?(dir_name) + end - all_dirs.each do |dir_path| + all_dirs.each do |dir_path| - if remove_flag - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".color(:red) + if remove_flag + if FileUtils.rm_rf dir_path + puts "Removed...#{dir_path}".color(:red) + else + puts "Cannot remove #{dir_path}".color(:red) + end else - puts "Cannot remove #{dir_path}".color(:red) + puts "Can be removed: #{dir_path}".color(:red) end - else - puts "Can be removed: #{dir_path}".color(:red) end end @@ -48,20 +48,21 @@ namespace :gitlab do warn_user_is_not_gitlab move_suffix = "+orphaned+#{Time.now.to_i}" - repo_root = Gitlab.config.gitlab_shell.repos_path - # Look for global repos (legacy, depth 1) and normal repos (depth 2) - IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| - find.each_line do |path| - path.chomp! - repo_with_namespace = path. - sub(repo_root, ''). - sub(%r{^/*}, ''). - chomp('.git'). - chomp('.wiki') - next if Project.find_with_namespace(repo_with_namespace) - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - File.rename(path, new_path) + Gitlab.config.repositories.storages.each do |name, repo_root| + # Look for global repos (legacy, depth 1) and normal repos (depth 2) + IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| + find.each_line do |path| + path.chomp! + repo_with_namespace = path. + sub(repo_root, ''). + sub(%r{^/*}, ''). + chomp('.git'). + chomp('.wiki') + next if Project.find_with_namespace(repo_with_namespace) + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + File.rename(path, new_path) + end end end end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 4753f00c26a..dbdd4e977e8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -2,73 +2,73 @@ namespace :gitlab do namespace :import do # How to use: # - # 1. copy the bare repos under the repos_path (commonly /home/git/repositories) + # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories) # 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production # # Notes: # * The project owner will set to the first administator of the system # * Existing projects will be skipped # - desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance" + desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do + Gitlab.config.repositories.storages.each do |name, git_base_path| + repos_to_import = Dir.glob(git_base_path + '/**/*.git') - git_base_path = Gitlab.config.gitlab_shell.repos_path - repos_to_import = Dir.glob(git_base_path + '/**/*.git') + repos_to_import.each do |repo_path| + # strip repo base path + repo_path[0..git_base_path.length] = '' - repos_to_import.each do |repo_path| - # strip repo base path - repo_path[0..git_base_path.length] = '' + path = repo_path.sub(/\.git$/, '') + group_name, name = File.split(path) + group_name = nil if group_name == '.' - path = repo_path.sub(/\.git$/, '') - group_name, name = File.split(path) - group_name = nil if group_name == '.' + puts "Processing #{repo_path}".color(:yellow) - puts "Processing #{repo_path}".color(:yellow) - - if path.end_with?('.wiki') - puts " * Skipping wiki repo" - next - end + if path.end_with?('.wiki') + puts " * Skipping wiki repo" + next + end - project = Project.find_with_namespace(path) + project = Project.find_with_namespace(path) - if project - puts " * #{project.name} (#{repo_path}) exists" - else - user = User.admins.reorder("id").first + if project + puts " * #{project.name} (#{repo_path}) exists" + else + user = User.admins.reorder("id").first - project_params = { - name: name, - path: name - } + project_params = { + name: name, + path: name + } - # find group namespace - if group_name - group = Namespace.find_by(path: group_name) - # create group namespace - unless group - group = Group.new(:name => group_name) - group.path = group_name - group.owner = user - if group.save - puts " * Created Group #{group.name} (#{group.id})".color(:green) - else - puts " * Failed trying to create group #{group.name}".color(:red) + # find group namespace + if group_name + group = Namespace.find_by(path: group_name) + # create group namespace + unless group + group = Group.new(:name => group_name) + group.path = group_name + group.owner = user + if group.save + puts " * Created Group #{group.name} (#{group.id})".color(:green) + else + puts " * Failed trying to create group #{group.name}".color(:red) + end end + # set project group + project_params[:namespace_id] = group.id end - # set project group - project_params[:namespace_id] = group.id - end - project = Projects::CreateService.new(user, project_params).execute + project = Projects::CreateService.new(user, project_params).execute - if project.persisted? - puts " * Created #{project.name} (#{repo_path})".color(:green) - project.update_repository_size - project.update_commit_count - else - puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) - puts " Errors: #{project.errors.messages}".color(:red) + if project.persisted? + puts " * Created #{project.name} (#{repo_path})".color(:green) + project.update_repository_size + project.update_commit_count + else + puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) + puts " Errors: #{project.errors.messages}".color(:red) + end end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 352b566df24..fe43d40e6d2 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -62,7 +62,10 @@ namespace :gitlab do puts "" puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" - puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}" + puts "Repository storage paths:" + Gitlab.config.repositories.storages.each do |name, path| + puts "- #{name}: \t#{path}" + end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index c7596e7abcb..ffcc76e5498 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -9,7 +9,7 @@ namespace :gitlab do scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end scope.find_each do |project| - base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + base = File.join(project.repository_storage_path, project.path_with_namespace) puts base + '.git' puts base + '.wiki.git' end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index b1648a4602a..c85ebdf8619 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -12,7 +12,6 @@ namespace :gitlab do gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') - repos_path = Gitlab.config.gitlab_shell.repos_path target_dir = Gitlab.config.gitlab_shell.path # Clone if needed @@ -35,7 +34,6 @@ namespace :gitlab do user: user, gitlab_url: gitlab_url, http_settings: {self_signed_cert: false}.stringify_keys, - repos_path: repos_path, auth_file: File.join(home_dir, ".ssh", "authorized_keys"), redis: { bin: %x{which redis-cli}.chomp, @@ -58,10 +56,10 @@ namespace :gitlab do File.open("config.yml", "w+") {|f| f.puts config.to_yaml} # Launch installation process - system(*%W(bin/install)) + system(*%W(bin/install) + repository_storage_paths_args) # (Re)create hooks - system(*%W(bin/create-hooks)) + system(*%W(bin/create-hooks) + repository_storage_paths_args) end # Required for debian packaging with PKGR: Setup .ssh/environment with @@ -73,6 +71,8 @@ namespace :gitlab do File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f| f.puts "PATH=#{ENV['PATH']}" end + + Gitlab::Shell.new.generate_and_link_secret_token end desc "GitLab | Setup gitlab-shell" @@ -87,7 +87,8 @@ namespace :gitlab do if File.exists?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.path_with_namespace) + if Gitlab::Shell.new.add_repository(project.repository_storage_path, + project.path_with_namespace) print '.' else print 'F' @@ -138,4 +139,3 @@ namespace :gitlab do system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) end end - diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index d0c019044b7..ab96b1d3593 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -125,10 +125,16 @@ namespace :gitlab do end def all_repos - IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp + Gitlab.config.repositories.storages.each do |name, path| + IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end end end end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end |