diff options
| author | Tomasz Maczukin <tomasz@maczukin.pl> | 2016-07-02 23:58:21 +0300 |
|---|---|---|
| committer | Tomasz Maczukin <tomasz@maczukin.pl> | 2016-07-02 23:58:21 +0300 |
| commit | f1a85747ca94cfe0aea25416da1e2ba498b636be (patch) | |
| tree | fd0ae2b443ee90b90fa61817be7e7ee23059c6c9 /lib | |
| parent | 3a6ebb1fd624c216a4ce65380e64072793b7ccda (diff) | |
| parent | fc3402b7eaa9e9839ac2b7087d9582f518b0bf96 (diff) | |
Merge branch 'master' into dev-master
* master: (98 commits)
Enable Style/EmptyLines cop, remove redundant ones
Update CHANGELOG
Cache results from jQuery selectors to retrieve namespace name
Fix import button when import fail due the namespace already been taken
Fix snippets comments not displayed
Fix emoji paths in relative root configurations
Exclude requesters from Project#members, Group#members and User#members
Upgrade Thin from 1.6.1 to 1.7.0.
Many squashed commits
Cache autocomplete results
Upgrade Sidekiq from 4.1.2 to 4.1.4.
Upgrade seed-fu from 2.3.5 to 2.3.6
use has_many relationship with events
Support creating a todo on issuables via API
Expose target, filter by state as string
Add todos API documentation and changelog
Improve the request / withdraw access button
Metrics for Rouge::Plugins::Redcarpet and Rouge::Formatters::HTMLGitlab
Groundwork for Kerberos SPNEGO (EE feature)
Update CHANGELOG 8.9.5 for runners related fixes
...
Diffstat (limited to 'lib')
64 files changed, 517 insertions, 329 deletions
diff --git a/lib/api/api.rb b/lib/api/api.rb index c3fff8b2f8f..3d7d67510a8 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -58,6 +58,7 @@ module API mount ::API::SystemHooks mount ::API::Tags mount ::API::Templates + mount ::API::Todos mount ::API::Triggers mount ::API::Users mount ::API::Variables diff --git a/lib/api/award_emoji.rb b/lib/api/award_emoji.rb index 985590312e3..c4fa1838b5a 100644 --- a/lib/api/award_emoji.rb +++ b/lib/api/award_emoji.rb @@ -11,7 +11,6 @@ module API [ ":id/#{awardable_string}/:#{awardable_id_string}/award_emoji", ":id/#{awardable_string}/:#{awardable_id_string}/notes/:note_id/award_emoji" ].each do |endpoint| - # Get a list of project +awardable+ award emoji # # Parameters: diff --git a/lib/api/branches.rb b/lib/api/branches.rb index 231840148d9..d467eb9d474 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -25,7 +25,7 @@ module API # branch (required) - The name of the branch # Example Request: # GET /projects/:id/repository/branches/:branch - get ':id/repository/branches/:branch', requirements: { branch: /.*/ } do + get ':id/repository/branches/:branch', requirements: { branch: /.+/ } do @branch = user_project.repository.branches.find { |item| item.name == params[:branch] } not_found!("Branch") unless @branch present @branch, with: Entities::RepoObject, project: user_project @@ -39,8 +39,7 @@ module API # Example Request: # PUT /projects/:id/repository/branches/:branch/protect put ':id/repository/branches/:branch/protect', - requirements: { branch: /.*/ } do - + requirements: { branch: /.+/ } do authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) @@ -59,8 +58,7 @@ module API # Example Request: # PUT /projects/:id/repository/branches/:branch/unprotect put ':id/repository/branches/:branch/unprotect', - requirements: { branch: /.*/ } do - + requirements: { branch: /.+/ } do authorize_admin_project @branch = user_project.repository.find_branch(params[:branch]) @@ -101,7 +99,7 @@ module API # Example Request: # DELETE /projects/:id/repository/branches/:branch delete ":id/repository/branches/:branch", - requirements: { branch: /.*/ } do + requirements: { branch: /.+/ } do authorize_push_project result = DeleteBranchService.new(user_project, current_user). execute(params[:branch]) diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 086d8511e8f..d36047acd1f 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -13,7 +13,6 @@ module API # Example Request: # GET /projects/:id/builds get ':id/builds' do - builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 5a23a18fe9c..8cc4368b5c2 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -272,6 +272,31 @@ module API expose :id, :project_id, :group_id, :group_access end + class Todo < Grape::Entity + expose :id + expose :project, using: Entities::BasicProjectDetails + expose :author, using: Entities::UserBasic + expose :action_name + expose :target_type + + expose :target do |todo, options| + Entities.const_get(todo.target_type).represent(todo.target, options) + end + + expose :target_url do |todo, options| + target_type = todo.target_type.underscore + target_url = "namespace_project_#{target_type}_url" + target_anchor = "note_#{todo.note_id}" if todo.note_id? + + Gitlab::Application.routes.url_helpers.public_send(target_url, + todo.project.namespace, todo.project, todo.target, anchor: target_anchor) + end + + expose :body + expose :state + expose :created_at + end + class Namespace < Grape::Entity expose :id, :path, :kind end @@ -376,6 +401,7 @@ module API expose :user_oauth_applications expose :after_sign_out_path expose :container_registry_token_expire_delay + expose :repository_storage end class Release < Grape::Entity diff --git a/lib/api/group_members.rb b/lib/api/group_members.rb index ab9b7c602b5..dbe5bb08d3f 100644 --- a/lib/api/group_members.rb +++ b/lib/api/group_members.rb @@ -77,7 +77,7 @@ module API member = group.group_members.find_by(user_id: params[:user_id]) if member.nil? - render_api_error!("404 Not Found - user_id:#{params[:user_id]} not a member of group #{group.name}",404) + render_api_error!("404 Not Found - user_id:#{params[:user_id]} not a member of group #{group.name}", 404) else member.destroy end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 1d361569d59..b32503e8516 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -20,6 +20,20 @@ module API @wiki ||= params[:project].end_with?('.wiki') && !Project.find_with_namespace(params[:project]) end + + def project + @project ||= begin + project_path = params[:project] + + # Check for *.wiki repositories. + # Strip out the .wiki from the pathname before finding the + # project. This applies the correct project permissions to + # the wiki repository as well. + project_path.chomp!('.wiki') if wiki? + + Project.find_with_namespace(project_path) + end + end end post "/allowed" do @@ -32,16 +46,6 @@ module API User.find_by(id: params[:user_id]) end - project_path = params[:project] - - # Check for *.wiki repositories. - # Strip out the .wiki from the pathname before finding the - # project. This applies the correct project permissions to - # the wiki repository as well. - project_path.chomp!('.wiki') if wiki? - - project = Project.find_with_namespace(project_path) - access = if wiki? Gitlab::GitAccessWiki.new(actor, project) @@ -49,7 +53,17 @@ module API Gitlab::GitAccess.new(actor, project) end - access.check(params[:action], params[:changes]) + access_status = access.check(params[:action], params[:changes]) + + response = { status: access_status.status, message: access_status.message } + + if access_status.status + # Return the repository full path so that gitlab-shell has it when + # handling ssh commands + response[:repository_path] = project.repository.path_to_repo + end + + response end # diff --git a/lib/api/milestones.rb b/lib/api/milestones.rb index 132043cf3f7..7a0cb7c99f3 100644 --- a/lib/api/milestones.rb +++ b/lib/api/milestones.rb @@ -115,7 +115,6 @@ module API issues = IssuesFinder.new(current_user, finder_params).execute present paginate(issues), with: Entities::Issue, current_user: current_user end - end end end diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index ccca65cbe1c..6bb70bc8bc3 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -28,7 +28,6 @@ module API present @hook, with: Entities::ProjectHook end - # Add hook to project # # Parameters: diff --git a/lib/api/project_members.rb b/lib/api/project_members.rb index b703da0557a..6a0b3e7d134 100644 --- a/lib/api/project_members.rb +++ b/lib/api/project_members.rb @@ -4,7 +4,6 @@ module API before { authenticate! } resource :projects do - # Get a project team members # # Parameters: diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 5a22d14988f..0cc1edd65c8 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -341,7 +341,6 @@ module API else not_found!("Source Project") end - end # Remove a forked_from relationship @@ -418,7 +417,6 @@ module API present paginate(projects), with: Entities::Project end - # Get a users list # # Example Request: diff --git a/lib/api/services.rb b/lib/api/services.rb index 203f04a6259..fc8598daa32 100644 --- a/lib/api/services.rb +++ b/lib/api/services.rb @@ -4,7 +4,6 @@ module API before { authenticate! } before { authorize_admin_project } - resource :projects do # Set <service_slug> service for project # diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 3e1ed3fe5c7..7b675e05fbb 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -61,7 +61,7 @@ module API # tag_name (required) - The name of the tag # Example Request: # DELETE /projects/:id/repository/tags/:tag - delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.*/ } do + delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do authorize_push_project result = DeleteTagService.new(user_project, current_user). execute(params[:tag_name]) @@ -83,7 +83,7 @@ module API # description (required) - Release notes with markdown support # Example Request: # POST /projects/:id/repository/tags/:tag_name/release - post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.*/ } do + post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project required_attributes! [:description] result = CreateReleaseService.new(user_project, current_user). @@ -104,7 +104,7 @@ module API # description (required) - Release notes with markdown support # Example Request: # PUT /projects/:id/repository/tags/:tag_name/release - put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.*/ } do + put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project required_attributes! [:description] result = UpdateReleaseService.new(user_project, current_user). diff --git a/lib/api/todos.rb b/lib/api/todos.rb new file mode 100644 index 00000000000..2a6bfa98ca4 --- /dev/null +++ b/lib/api/todos.rb @@ -0,0 +1,82 @@ +module API + # Todos API + class Todos < Grape::API + before { authenticate! } + + ISSUABLE_TYPES = { + 'merge_requests' => ->(id) { user_project.merge_requests.find(id) }, + 'issues' => ->(id) { find_project_issue(id) } + } + + resource :projects do + ISSUABLE_TYPES.each do |type, finder| + type_id_str = "#{type.singularize}_id".to_sym + + # Create a todo on an issuable + # + # Parameters: + # id (required) - The ID of a project + # issuable_id (required) - The ID of an issuable + # Example Request: + # POST /projects/:id/issues/:issuable_id/todo + # POST /projects/:id/merge_requests/:issuable_id/todo + post ":id/#{type}/:#{type_id_str}/todo" do + issuable = instance_exec(params[type_id_str], &finder) + todo = TodoService.new.mark_todo(issuable, current_user).first + + if todo + present todo, with: Entities::Todo, current_user: current_user + else + not_modified! + end + end + end + end + + resource :todos do + helpers do + def find_todos + TodosFinder.new(current_user, params).execute + end + end + + # Get a todo list + # + # Example Request: + # GET /todos + # + get do + todos = find_todos + + present paginate(todos), with: Entities::Todo, current_user: current_user + end + + # Mark a todo as done + # + # Parameters: + # id: (required) - The ID of the todo being marked as done + # + # Example Request: + # DELETE /todos/:id + # + delete ':id' do + todo = current_user.todos.find(params[:id]) + todo.done + + present todo, with: Entities::Todo, current_user: current_user + end + + # Mark all todos as done + # + # Example Request: + # DELETE /todos + # + delete do + todos = find_todos + todos.each(&:done) + + present paginate(Kaminari.paginate_array(todos)), with: Entities::Todo, current_user: current_user + end + end + end +end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 7b91215d50b..b9773f98d75 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -2,8 +2,6 @@ require 'yaml' module Backup class Repository - attr_reader :repos_path - def dump prepare @@ -50,10 +48,12 @@ module Backup end def restore - if File.exists?(repos_path) + Gitlab.config.repositories.storages.each do |name, path| + next unless File.exists?(path) + # Move repos dir to 'repositories.old' dir - bk_repos_path = File.join(repos_path, '..', 'repositories.old.' + Time.now.to_i.to_s) - FileUtils.mv(repos_path, bk_repos_path) + bk_repos_path = File.join(path, '..', 'repositories.old.' + Time.now.to_i.to_s) + FileUtils.mv(path, bk_repos_path) end FileUtils.mkdir_p(repos_path) @@ -61,7 +61,7 @@ module Backup Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " - project.namespace.ensure_dir_exist if project.namespace + project.ensure_dir_exist if File.exists?(path_to_bundle(project)) FileUtils.mkdir_p(path_to_repo(project)) @@ -100,8 +100,8 @@ module Backup end $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) - cmd = "#{Gitlab.config.gitlab_shell.path}/bin/create-hooks" - if system(cmd) + cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args + if system(*cmd) $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) @@ -120,10 +120,6 @@ module Backup File.join(backup_repos_path, project.path_with_namespace + ".bundle") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - def backup_repos_path File.join(Gitlab.config.backup.path, "repositories") end @@ -139,5 +135,11 @@ module Backup def silent {err: '/dev/null', out: '/dev/null'} end + + private + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end end diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 81d66271136..d77a5e3ff09 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -160,11 +160,7 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attribute( - original: link_text || match, - project: project.id, - object_sym => object.id - ) + data = data_attributes_for(link_text || match, project, object) if matches.names.include?("url") && matches[:url] url = matches[:url] @@ -183,6 +179,14 @@ module Banzai end end + def data_attributes_for(text, project, object) + data_attribute( + original: text, + project: project.id, + object_sym => object.id + ) + end + def object_link_text_extras(object, matches) extras = [] diff --git a/lib/banzai/filter/image_link_filter.rb b/lib/banzai/filter/image_link_filter.rb index 8aa6f8f124a..f0fb6084a35 100644 --- a/lib/banzai/filter/image_link_filter.rb +++ b/lib/banzai/filter/image_link_filter.rb @@ -8,7 +8,6 @@ module Banzai # of the anchor, and then replace the img with the link-wrapped version. def call doc.xpath('descendant-or-self::img[not(ancestor::a)]').each do |img| - div = doc.document.create_element( 'div', class: 'image-container' diff --git a/lib/banzai/filter/issue_reference_filter.rb b/lib/banzai/filter/issue_reference_filter.rb index 5351272f42d..4042e9a4c25 100644 --- a/lib/banzai/filter/issue_reference_filter.rb +++ b/lib/banzai/filter/issue_reference_filter.rb @@ -46,6 +46,26 @@ module Banzai end end + def object_link_title(object) + if object.is_a?(ExternalIssue) + "Issue in #{object.project.external_issue_tracker.title}" + else + super + end + end + + def data_attributes_for(text, project, object) + if object.is_a?(ExternalIssue) + data_attribute( + project: project.id, + external_issue: object.id, + reference_type: ExternalIssueReferenceFilter.reference_type + ) + else + super + end + end + def find_projects_for_paths(paths) super(paths).includes(:gitlab_issue_tracker_service) end diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 2d6f34c9cd8..bf058241cda 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -29,7 +29,7 @@ module Banzai def data_attribute(attributes = {}) attributes = attributes.reject { |_, v| v.nil? } - attributes[:reference_type] = self.class.reference_type + attributes[:reference_type] ||= self.class.reference_type attributes.delete(:original) if context[:no_original_data] attributes.map { |key, value| %Q(data-#{key.to_s.dasherize}="#{escape_once(value)}") }.join(" ") end diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 62a79c62e20..536b478979f 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -27,12 +27,17 @@ module Banzai highlighted = "<pre>#{code}</pre>" end - # Replace the parent `pre` element with the entire highlighted block - node.parent.replace(highlighted) + # Extracted to a method to measure it + replace_parent_pre_element(node, highlighted) end private + def replace_parent_pre_element(node, highlighted) + # Replace the parent `pre` element with the entire highlighted block + node.parent.replace(highlighted) + end + # Override Rouge::Plugins::Redcarpet#rouge_formatter def rouge_formatter(lexer) Rouge::Formatters::HTMLGitlab.new( diff --git a/lib/banzai/filter/wiki_link_filter.rb b/lib/banzai/filter/wiki_link_filter.rb index 1bb6d6bba87..269d5bf74fa 100644 --- a/lib/banzai/filter/wiki_link_filter.rb +++ b/lib/banzai/filter/wiki_link_filter.rb @@ -8,7 +8,6 @@ module Banzai # Context options: # :project_wiki class WikiLinkFilter < HTML::Pipeline::Filter - def call return doc unless project_wiki? diff --git a/lib/banzai/pipeline/full_pipeline.rb b/lib/banzai/pipeline/full_pipeline.rb index d47ddfda4be..3c974f73176 100644 --- a/lib/banzai/pipeline/full_pipeline.rb +++ b/lib/banzai/pipeline/full_pipeline.rb @@ -1,7 +1,6 @@ module Banzai module Pipeline class FullPipeline < CombinedPipeline.new(PlainMarkdownPipeline, GfmPipeline) - end end end diff --git a/lib/ci/charts.rb b/lib/ci/charts.rb index 5270108ef0f..1d7126a432d 100644 --- a/lib/ci/charts.rb +++ b/lib/ci/charts.rb @@ -13,7 +13,6 @@ module Ci collect end - def push(from, to, format) @labels << from.strftime(format) @total << project.builds. diff --git a/lib/disable_email_interceptor.rb b/lib/disable_email_interceptor.rb index 1b80be112a4..cee664b8951 100644 --- a/lib/disable_email_interceptor.rb +++ b/lib/disable_email_interceptor.rb @@ -1,6 +1,5 @@ # Read about interceptors in http://guides.rubyonrails.org/action_mailer_basics.html#intercepting-emails class DisableEmailInterceptor - def self.delivering_email(message) message.perform_deliveries = false Rails.logger.info "Emails disabled! Interceptor prevented sending mail #{message.subject}" diff --git a/lib/gitlab.rb b/lib/gitlab.rb index 37f4c34054f..c3064163e07 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -2,6 +2,11 @@ require_dependency 'gitlab/git' module Gitlab def self.com? - Gitlab.config.gitlab.url == 'https://gitlab.com' + # Check `staging?` as well to keep parity with gitlab.com + Gitlab.config.gitlab.url == 'https://gitlab.com' || staging? + end + + def self.staging? + Gitlab.config.gitlab.url == 'https://staging.gitlab.com' end end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index 0b9c2e730f9..1a22ad9acf5 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -4,7 +4,6 @@ module Gitlab # Parser/renderer for the AsciiDoc format that uses Asciidoctor and filters # the resulting HTML through HTML pipeline filters. module Asciidoc - DEFAULT_ADOC_ATTRS = [ 'showtitle', 'idprefix=user-content-', 'idseparator=-', 'env=gitlab', 'env-gitlab', 'source-highlighter=html-pipeline' diff --git a/lib/gitlab/award_emoji.rb b/lib/gitlab/award_emoji.rb index 51b1df9ecbd..c94bfc0e65f 100644 --- a/lib/gitlab/award_emoji.rb +++ b/lib/gitlab/award_emoji.rb @@ -66,8 +66,17 @@ module Gitlab def self.urls @urls ||= begin path = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') + # Construct the full asset path ourselves because + # ActionView::Helpers::AssetUrlHelper.asset_url is slow for hundreds + # of entries since it has to do a lot of extra work (e.g. regexps). prefix = Gitlab::Application.config.assets.prefix digest = Gitlab::Application.config.assets.digest + base = + if defined?(Gitlab::Application.config.relative_url_root) && Gitlab::Application.config.relative_url_root + Gitlab::Application.config.relative_url_root + else + '' + end JSON.parse(File.read(path)).map do |hash| if digest @@ -76,7 +85,7 @@ module Gitlab fname = hash['unicode'] end - { name: hash['name'], path: "#{prefix}/#{fname}.png" } + { name: hash['name'], path: File.join(base, prefix, "#{fname}.png") } end end end diff --git a/lib/gitlab/backend/grack_auth.rb b/lib/gitlab/backend/grack_auth.rb index ab7b811c5d8..478f145bfed 100644 --- a/lib/gitlab/backend/grack_auth.rb +++ b/lib/gitlab/backend/grack_auth.rb @@ -8,7 +8,6 @@ module Grack end class Auth < Rack::Auth::Basic - attr_accessor :user, :project, :env def call(env) @@ -22,7 +21,7 @@ module Grack # Need this if under RELATIVE_URL_ROOT unless Gitlab.config.gitlab.relative_url_root.empty? # If website is mounted using relative_url_root need to remove it first - @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root,'') + @env['PATH_INFO'] = @request.path.sub(Gitlab.config.gitlab.relative_url_root, '') else @env['PATH_INFO'] = @request.path end diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb index 3e3986d6382..34e0143a82e 100644 --- a/lib/gitlab/backend/shell.rb +++ b/lib/gitlab/backend/shell.rb @@ -1,3 +1,5 @@ +require 'securerandom' + module Gitlab class Shell class Error < StandardError; end @@ -18,77 +20,82 @@ module Gitlab # Init new repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # add_repository("gitlab/gitlab-ci") + # add_repository("/path/to/storage", "gitlab/gitlab-ci") # - def add_repository(name) + def add_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'add-project', "#{name}.git"]) + 'add-project', storage, "#{name}.git"]) end # Import repository # + # storage - project's storage path # name - project path with namespace # # Ex. - # import_repository("gitlab/gitlab-ci", "https://github.com/randx/six.git") + # import_repository("/path/to/storage", "gitlab/gitlab-ci", "https://github.com/randx/six.git") # - def import_repository(name, url) - output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', "#{name}.git", url, '900']) + def import_repository(storage, name, url) + output, status = Popen::popen([gitlab_shell_projects_path, 'import-project', + storage, "#{name}.git", url, '900']) raise Error, output unless status.zero? true end # Move repository - # + # storage - project's storage path # path - project path with namespace # new_path - new project path with namespace # # Ex. - # mv_repository("gitlab/gitlab-ci", "randx/gitlab-ci-new") + # mv_repository("/path/to/storage", "gitlab/gitlab-ci", "randx/gitlab-ci-new") # - def mv_repository(path, new_path) + def mv_repository(storage, path, new_path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'mv-project', - "#{path}.git", "#{new_path}.git"]) + storage, "#{path}.git", "#{new_path}.git"]) end # Fork repository to new namespace - # + # storage - project's storage path # path - project path with namespace # fork_namespace - namespace for forked project # # Ex. - # fork_repository("gitlab/gitlab-ci", "randx") + # fork_repository("/path/to/storage", "gitlab/gitlab-ci", "randx") # - def fork_repository(path, fork_namespace) + def fork_repository(storage, path, fork_namespace) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'fork-project', - "#{path}.git", fork_namespace]) + storage, "#{path}.git", fork_namespace]) end # Remove repository from file system # + # storage - project's storage path # name - project path with namespace # # Ex. - # remove_repository("gitlab/gitlab-ci") + # remove_repository("/path/to/storage", "gitlab/gitlab-ci") # - def remove_repository(name) + def remove_repository(storage, name) Gitlab::Utils.system_silent([gitlab_shell_projects_path, - 'rm-project', "#{name}.git"]) + 'rm-project', storage, "#{name}.git"]) end # Gc repository # + # storage - project storage path # path - project path with namespace # # Ex. - # gc("gitlab/gitlab-ci") + # gc("/path/to/storage", "gitlab/gitlab-ci") # - def gc(path) + def gc(storage, path) Gitlab::Utils.system_silent([gitlab_shell_projects_path, 'gc', - "#{path}.git"]) + storage, "#{path}.git"]) end # Add new key to gitlab-shell @@ -133,31 +140,31 @@ module Gitlab # Add empty directory for storing repositories # # Ex. - # add_namespace("gitlab") + # add_namespace("/path/to/storage", "gitlab") # - def add_namespace(name) - FileUtils.mkdir(full_path(name), mode: 0770) unless exists?(name) + def add_namespace(storage, name) + FileUtils.mkdir(full_path(storage, name), mode: 0770) unless exists?(storage, name) end # Remove directory from repositories storage # Every repository inside this directory will be removed too # # Ex. - # rm_namespace("gitlab") + # rm_namespace("/path/to/storage", "gitlab") # - def rm_namespace(name) - FileUtils.rm_r(full_path(name), force: true) + def rm_namespace(storage, name) + FileUtils.rm_r(full_path(storage, name), force: true) end # Move namespace directory inside repositories storage # # Ex. - # mv_namespace("gitlab", "gitlabhq") + # mv_namespace("/path/to/storage", "gitlab", "gitlabhq") # - def mv_namespace(old_name, new_name) - return false if exists?(new_name) || !exists?(old_name) + def mv_namespace(storage, old_name, new_name) + return false if exists?(storage, new_name) || !exists?(storage, old_name) - FileUtils.mv(full_path(old_name), full_path(new_name)) + FileUtils.mv(full_path(storage, old_name), full_path(storage, new_name)) end def url_to_repo(path) @@ -176,11 +183,26 @@ module Gitlab # Check if such directory exists in repositories. # # Usage: - # exists?('gitlab') - # exists?('gitlab/cookies.git') + # exists?(storage, 'gitlab') + # exists?(storage, 'gitlab/cookies.git') # - def exists?(dir_name) - File.exist?(full_path(dir_name)) + def exists?(storage, dir_name) + File.exist?(full_path(storage, dir_name)) + end + + # Create (if necessary) and link the secret token file + def generate_and_link_secret_token + secret_file = Gitlab.config.gitlab_shell.secret_file + unless File.exist? secret_file + # Generate a new token of 16 random hexadecimal characters and store it in secret_file. + token = SecureRandom.hex(16) + File.write(secret_file, token) + end + + link_path = File.join(gitlab_shell_path, '.gitlab_shell_secret') + if File.exist?(gitlab_shell_path) && !File.exist?(link_path) + FileUtils.symlink(secret_file, link_path) + end end protected @@ -193,14 +215,10 @@ module Gitlab File.expand_path("~#{Gitlab.config.gitlab_shell.ssh_user}") end - def repos_path - Gitlab.config.gitlab_shell.repos_path - end - - def full_path(dir_name) + def full_path(storage, dir_name) raise ArgumentError.new("Directory name can't be blank") if dir_name.blank? - File.join(repos_path, dir_name) + File.join(storage, dir_name) end def gitlab_shell_projects_path diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 28c34429c1f..54b46e5d23f 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -9,10 +9,14 @@ module Gitlab end def ensure_application_settings! - settings = ::ApplicationSetting.cached + if connect_to_db? + begin + settings = ::ApplicationSetting.current + # In case Redis isn't running or the Redis UNIX socket file is not available + rescue ::Redis::BaseError, ::Errno::ENOENT + settings = ::ApplicationSetting.last + end - if !settings && connect_to_db? - settings = ::ApplicationSetting.current settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration? end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 522dd2b9428..59a2367b65d 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -40,7 +40,6 @@ module Gitlab line_obj_index += 1 end - case line[0] when "+" line_new += 1 diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index d325eca6d99..043f10d96a9 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -4,26 +4,39 @@ module Gitlab GITHUB_SAFE_REMAINING_REQUESTS = 100 GITHUB_SAFE_SLEEP_TIME = 500 - attr_reader :client, :api + attr_reader :access_token def initialize(access_token) - @client = ::OAuth2::Client.new( - config.app_id, - config.app_secret, - github_options.merge(ssl: { verify: config['verify_ssl'] }) - ) + @access_token = access_token if access_token ::Octokit.auto_paginate = false + end + end + + def api + @api ||= ::Octokit::Client.new( + access_token: access_token, + api_endpoint: github_options[:site], + # If there is no config, we're connecting to github.com and we + # should verify ssl. + connection_options: { + ssl: { verify: config ? config['verify_ssl'] : true } + } + ) + end - @api = ::Octokit::Client.new( - access_token: access_token, - api_endpoint: github_options[:site], - connection_options: { - ssl: { verify: config['verify_ssl'] } - } - ) + def client + unless config + raise Projects::ImportService::Error, + 'OAuth configuration for GitHub missing.' end + + @client ||= ::OAuth2::Client.new( + config.app_id, + config.app_secret, + github_options.merge(ssl: { verify: config['verify_ssl'] }) + ) end def authorize_url(redirect_uri) @@ -56,7 +69,11 @@ module Gitlab end def github_options - config["args"]["client_options"].deep_symbolize_keys + if config + config["args"]["client_options"].deep_symbolize_keys + else + OmniAuth::Strategies::GitHub.default_options[:client_options].symbolize_keys + end end def rate_limit diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 2286ac8829c..730978d502b 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -167,7 +167,7 @@ module Gitlab def import_wiki unless project.wiki_enabled? wiki = WikiFormatter.new(project) - gitlab_shell.import_repository(wiki.path_with_namespace, wiki.import_url) + gitlab_shell.import_repository(project.repository_storage_path, wiki.path_with_namespace, wiki.import_url) project.update_attribute(:wiki_enabled, true) end diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb index 99cf85d9a3b..588647e5adb 100644 --- a/lib/gitlab/import_export.rb +++ b/lib/gitlab/import_export.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport extend self - VERSION = '0.1.0' + VERSION = '0.1.1' def export_path(relative_path:) File.join(storage_path, relative_path) diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/import_export.yml index 164ab6238c4..05f4ad527ac 100644 --- a/lib/gitlab/import_export/import_export.yml +++ b/lib/gitlab/import_export/import_export.yml @@ -1,24 +1,29 @@ # Model relationships to be included in the project import/export project_tree: - issues: + - :events - notes: - :author + - :author + - :events - :labels - - :milestones + - milestones: + - :events - snippets: - notes: :author - :releases - - :events - project_members: - :user - merge_requests: - notes: - :author + - :author + - :events - :merge_request_diff + - :events - pipelines: - notes: - :author + - :author + - :events - :statuses - :variables - :triggers diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 595b20a09bd..8f66f48cbfe 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class Importer - def initialize(project) @archive_file = project.import_source @current_user = project.creator diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index c569a35a48b..b459054c198 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class MembersMapper - attr_reader :missing_author_ids def initialize(exported_members:, user:, project:) diff --git a/lib/gitlab/import_export/project_creator.rb b/lib/gitlab/import_export/project_creator.rb index 89388d1984b..77bb3ca6581 100644 --- a/lib/gitlab/import_export/project_creator.rb +++ b/lib/gitlab/import_export/project_creator.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class ProjectCreator - def initialize(namespace_id, current_user, file, project_path) @namespace_id = namespace_id @current_user = current_user diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index dd71b92c522..0ac6ff01e3b 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class ProjectTreeRestorer - def initialize(user:, shared:, project:) @path = File.join(shared.export_path, 'project.json') @user = user diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 19defd8f03a..15f5dd31035 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class Reader - attr_reader :tree def initialize(shared:) @@ -55,7 +54,6 @@ module Gitlab @json_config_hash end - # If the model is a hash, process the sub_models, which could also be hashes # If there is a list, add to an existing array, otherwise use hash syntax # +current_key+ main model that will be a key in the hash diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index 92bf7e0a2fc..9824df3f274 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class RelationFactory - OVERRIDES = { snippets: :project_snippets, pipelines: 'Ci::Pipeline', statuses: 'commit_status', @@ -33,6 +32,7 @@ module Gitlab update_user_references update_project_references reset_ci_tokens if @relation_name == 'Ci::Trigger' + @relation_hash['data'].deep_symbolize_keys! if @relation_name == :events && @relation_hash['data'] generate_imported_object end diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 6aff05b886a..5d6de8bc475 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class Shared - attr_reader :errors, :opts def initialize(opts) diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index 7292e9d9712..d6f4fa57510 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class UploadsSaver - def initialize(project:, shared:) @project = project @shared = shared diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index cf5c62c5e3c..abfc694b879 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class VersionChecker - def self.check!(*args) new(*args).check! end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index f7f73dc9343..9b642d740b7 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -1,7 +1,6 @@ module Gitlab module ImportExport class VersionSaver - def initialize(shared:) @shared = shared end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index 948d43582cf..59a05411fe9 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -24,8 +24,6 @@ module Gitlab 'GitLab export' => 'gitlab_project' } end - end - end end diff --git a/lib/gitlab/key_fingerprint.rb b/lib/gitlab/key_fingerprint.rb index 8684b4636ea..b75ae512d92 100644 --- a/lib/gitlab/key_fingerprint.rb +++ b/lib/gitlab/key_fingerprint.rb @@ -39,7 +39,7 @@ module Gitlab # OpenSSH 6.8 introduces a new default output format for fingerprints. # Check the version and decide which command to use. - version_output, version_status = popen(%W(ssh -V)) + version_output, version_status = popen(%w(ssh -V)) return false unless version_status.zero? version_matches = version_output.match(/OpenSSH_(?<major>\d+)\.(?<minor>\d+)/) diff --git a/lib/gitlab/lfs/response.rb b/lib/gitlab/lfs/response.rb index e3ed2f6791d..811363405a8 100644 --- a/lib/gitlab/lfs/response.rb +++ b/lib/gitlab/lfs/response.rb @@ -1,7 +1,6 @@ module Gitlab module Lfs class Response - def initialize(project, user, ci, request) @origin_project = project @project = storage_project(project) diff --git a/lib/gitlab/o_auth/auth_hash.rb b/lib/gitlab/o_auth/auth_hash.rb index 36e5c2670bb..7d6911a1ab3 100644 --- a/lib/gitlab/o_auth/auth_hash.rb +++ b/lib/gitlab/o_auth/auth_hash.rb @@ -66,7 +66,7 @@ module Gitlab # Get the first part of the email address (before @) # In addtion in removes illegal characters def generate_username(email) - email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/,'').to_s + email.match(/^[^@]*/)[0].mb_chars.normalize(:kd).gsub(/[^\x00-\x7F]/, '').to_s end def generate_temporarily_email(username) diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index 746ec283330..4e2f8ed5587 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -1,7 +1,6 @@ module Gitlab # Parser/renderer for markups without other special support code. module OtherMarkup - # Public: Converts the provided markup into HTML. # # input - the source text in a markup format diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index c84c68f96f6..ffad5e17c78 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -13,7 +13,6 @@ module Gitlab "Cannot start with '-' or end in '.'." \ end - def namespace_name_regex @namespace_name_regex ||= /\A[\p{Alnum}\p{Pd}_\. ]*\z/.freeze end @@ -22,7 +21,6 @@ module Gitlab "can contain only letters, digits, '_', '.', dash and space." end - def project_name_regex @project_name_regex ||= /\A[\p{Alnum}_][\p{Alnum}\p{Pd}_\. ]*\z/.freeze end @@ -32,7 +30,6 @@ module Gitlab "It must start with letter, digit or '_'." end - def project_path_regex @project_path_regex ||= /\A[a-zA-Z0-9_.][a-zA-Z0-9_\-\.]*(?<!\.git|\.atom)\z/.freeze end @@ -42,7 +39,6 @@ module Gitlab "Cannot start with '-', end in '.git' or end in '.atom'" \ end - def file_name_regex @file_name_regex ||= /\A[a-zA-Z0-9_\-\.\@]*\z/.freeze end @@ -59,7 +55,6 @@ module Gitlab "can contain only letters, digits, '_', '-', '@' and '.'. Separate directories with a '/'. " end - def directory_traversal_regex @directory_traversal_regex ||= /\.{2}/.freeze end @@ -68,7 +63,6 @@ module Gitlab "cannot include directory traversal. " end - def archive_formats_regex # |zip|tar| tar.gz | tar.bz2 | @archive_formats_regex ||= /(zip|tar|tar\.gz|tgz|gz|tar\.bz2|tbz|tbz2|tb2|bz2)/.freeze diff --git a/lib/gitlab/saml/auth_hash.rb b/lib/gitlab/saml/auth_hash.rb index 32c1c9ec5bb..67a5f368bdb 100644 --- a/lib/gitlab/saml/auth_hash.rb +++ b/lib/gitlab/saml/auth_hash.rb @@ -1,7 +1,6 @@ module Gitlab module Saml class AuthHash < Gitlab::OAuth::AuthHash - def groups get_raw(Gitlab::Saml::Config.groups) end @@ -13,7 +12,6 @@ module Gitlab # otherwise just the first value is returned auth_hash.extra[:raw_info].all[key] end - end end end diff --git a/lib/gitlab/saml/config.rb b/lib/gitlab/saml/config.rb index 0f40c00f547..574c3a4b28c 100644 --- a/lib/gitlab/saml/config.rb +++ b/lib/gitlab/saml/config.rb @@ -1,7 +1,6 @@ module Gitlab module Saml class Config - class << self def options Gitlab.config.omniauth.providers.find { |provider| provider.name == 'saml' } @@ -15,7 +14,6 @@ module Gitlab options[:external_groups] end end - end end end diff --git a/lib/gitlab/saml/user.rb b/lib/gitlab/saml/user.rb index 8943022612c..f253dc7477e 100644 --- a/lib/gitlab/saml/user.rb +++ b/lib/gitlab/saml/user.rb @@ -6,7 +6,6 @@ module Gitlab module Saml class User < Gitlab::OAuth::User - def save super('SAML') end diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index 9ee72fde92f..b43ee5b3383 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -33,12 +33,13 @@ namespace :gitlab do unless backup.skipped?('db') unless ENV['force'] == 'yes' - warning = warning = <<-MSG.strip_heredoc + warning = <<-MSG.strip_heredoc Before restoring the database we recommend removing all existing tables to avoid future upgrade problems. Be aware that if you have custom tables in the GitLab database these tables and all data will be removed. MSG + puts warning.color(:red) ask_to_continue puts 'Removing all tables. Press `Ctrl-C` within 5 seconds to abort'.color(:yellow) sleep(5) diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 12d6ac45fb6..e9a4e37ec48 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -356,97 +356,108 @@ namespace :gitlab do ######################## def check_repo_base_exists - print "Repo base directory exists? ... " + puts "Repo base directory exists?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.exists?(repo_base_path) - puts "yes".color(:green) - else - puts "no".color(:red) - puts "#{repo_base_path} is missing".color(:red) - try_fixing_it( - "This should have been created when setting up GitLab Shell.", - "Make sure it's set correctly in config/gitlab.yml", - "Make sure GitLab Shell is installed correctly." - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + if File.exists?(repo_base_path) + puts "yes".color(:green) + else + puts "no".color(:red) + puts "#{repo_base_path} is missing".color(:red) + try_fixing_it( + "This should have been created when setting up GitLab Shell.", + "Make sure it's set correctly in config/gitlab.yml", + "Make sure GitLab Shell is installed correctly." + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_is_not_symlink - print "Repo base directory is a symlink? ... " + puts "Repo storage directories are symlinks?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - unless File.symlink?(repo_base_path) - puts "no".color(:green) - else - puts "yes".color(:red) - try_fixing_it( - "Make sure it's set to the real directory in config/gitlab.yml" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + unless File.symlink?(repo_base_path) + puts "no".color(:green) + else + puts "yes".color(:red) + try_fixing_it( + "Make sure it's set to the real directory in config/gitlab.yml" + ) + fix_and_rerun + end end end def check_repo_base_permissions - print "Repo base access is drwxrws---? ... " + puts "Repo paths access is drwxrws---?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") - puts "yes".color(:green) - else - puts "no".color(:red) - try_fixing_it( - "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", - "sudo chmod -R ug-s #{repo_base_path}", - "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + if File.stat(repo_base_path).mode.to_s(8).ends_with?("2770") + puts "yes".color(:green) + else + puts "no".color(:red) + try_fixing_it( + "sudo chmod -R ug+rwX,o-rwx #{repo_base_path}", + "sudo chmod -R ug-s #{repo_base_path}", + "sudo find #{repo_base_path} -type d -print0 | sudo xargs -0 chmod g+s" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end def check_repo_base_user_and_group gitlab_shell_ssh_user = Gitlab.config.gitlab_shell.ssh_user gitlab_shell_owner_group = Gitlab.config.gitlab_shell.owner_group - print "Repo base owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}? ... " + puts "Repo paths owned by #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group}?" - repo_base_path = Gitlab.config.gitlab_shell.repos_path - unless File.exists?(repo_base_path) - puts "can't check because of previous errors".color(:magenta) - return - end + Gitlab.config.repositories.storages.each do |name, repo_base_path| + print "#{name}... " - uid = uid_for(gitlab_shell_ssh_user) - gid = gid_for(gitlab_shell_owner_group) - if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid - puts "yes".color(:green) - else - puts "no".color(:red) - puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) - try_fixing_it( - "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" - ) - for_more_information( - see_installation_guide_section "GitLab Shell" - ) - fix_and_rerun + unless File.exists?(repo_base_path) + puts "can't check because of previous errors".color(:magenta) + return + end + + uid = uid_for(gitlab_shell_ssh_user) + gid = gid_for(gitlab_shell_owner_group) + if File.stat(repo_base_path).uid == uid && File.stat(repo_base_path).gid == gid + puts "yes".color(:green) + else + puts "no".color(:red) + puts " User id for #{gitlab_shell_ssh_user}: #{uid}. Groupd id for #{gitlab_shell_owner_group}: #{gid}".color(:blue) + try_fixing_it( + "sudo chown -R #{gitlab_shell_ssh_user}:#{gitlab_shell_owner_group} #{repo_base_path}" + ) + for_more_information( + see_installation_guide_section "GitLab Shell" + ) + fix_and_rerun + end end end @@ -473,7 +484,7 @@ namespace :gitlab do else puts "wrong or missing hooks".color(:red) try_fixing_it( - sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')}"), + sudo_gitlab("#{File.join(gitlab_shell_path, 'bin/create-hooks')} #{repository_storage_paths_args.join(' ')}"), 'Check the hooks_path in config/gitlab.yml', 'Check your gitlab-shell installation' ) @@ -785,13 +796,13 @@ namespace :gitlab do namespace :repo do desc "GitLab | Check the integrity of the repositories managed by GitLab" task check: :environment do - namespace_dirs = Dir.glob( - File.join(Gitlab.config.gitlab_shell.repos_path, '*') - ) + Gitlab.config.repositories.storages.each do |name, path| + namespace_dirs = Dir.glob(File.join(path, '*')) - namespace_dirs.each do |namespace_dir| - repo_dirs = Dir.glob(File.join(namespace_dir, '*')) - repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + namespace_dirs.each do |namespace_dir| + repo_dirs = Dir.glob(File.join(namespace_dir, '*')) + repo_dirs.each { |repo_dir| check_repo_integrity(repo_dir) } + end end end end @@ -799,12 +810,12 @@ namespace :gitlab do namespace :user do desc "GitLab | Check the integrity of a specific user's repositories" task :check_repos, [:username] => :environment do |t, args| - username = args[:username] || prompt("Check repository integrity for which username? ".color(:blue)) + username = args[:username] || prompt("Check repository integrity for fsername? ".color(:blue)) user = User.find_by(username: username) if user repo_dirs = user.authorized_projects.map do |p| File.join( - Gitlab.config.gitlab_shell.repos_path, + p.repository_storage_path, "#{p.path_with_namespace}.git" ) end diff --git a/lib/tasks/gitlab/cleanup.rake b/lib/tasks/gitlab/cleanup.rake index ab0028d6603..b7cbdc6cd78 100644 --- a/lib/tasks/gitlab/cleanup.rake +++ b/lib/tasks/gitlab/cleanup.rake @@ -5,36 +5,36 @@ namespace :gitlab do warn_user_is_not_gitlab remove_flag = ENV['REMOVE'] - namespaces = Namespace.pluck(:path) - git_base_path = Gitlab.config.gitlab_shell.repos_path - all_dirs = Dir.glob(git_base_path + '/*') + Gitlab.config.repositories.storages.each do |name, git_base_path| + all_dirs = Dir.glob(git_base_path + '/*') - puts git_base_path.color(:yellow) - puts "Looking for directories to remove... " + puts git_base_path.color(:yellow) + puts "Looking for directories to remove... " - all_dirs.reject! do |dir| - # skip if git repo - dir =~ /.git$/ - end + all_dirs.reject! do |dir| + # skip if git repo + dir =~ /.git$/ + end - all_dirs.reject! do |dir| - dir_name = File.basename dir + all_dirs.reject! do |dir| + dir_name = File.basename dir - # skip if namespace present - namespaces.include?(dir_name) - end + # skip if namespace present + namespaces.include?(dir_name) + end - all_dirs.each do |dir_path| + all_dirs.each do |dir_path| - if remove_flag - if FileUtils.rm_rf dir_path - puts "Removed...#{dir_path}".color(:red) + if remove_flag + if FileUtils.rm_rf dir_path + puts "Removed...#{dir_path}".color(:red) + else + puts "Cannot remove #{dir_path}".color(:red) + end else - puts "Cannot remove #{dir_path}".color(:red) + puts "Can be removed: #{dir_path}".color(:red) end - else - puts "Can be removed: #{dir_path}".color(:red) end end @@ -48,20 +48,21 @@ namespace :gitlab do warn_user_is_not_gitlab move_suffix = "+orphaned+#{Time.now.to_i}" - repo_root = Gitlab.config.gitlab_shell.repos_path - # Look for global repos (legacy, depth 1) and normal repos (depth 2) - IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| - find.each_line do |path| - path.chomp! - repo_with_namespace = path. - sub(repo_root, ''). - sub(%r{^/*}, ''). - chomp('.git'). - chomp('.wiki') - next if Project.find_with_namespace(repo_with_namespace) - new_path = path + move_suffix - puts path.inspect + ' -> ' + new_path.inspect - File.rename(path, new_path) + Gitlab.config.repositories.storages.each do |name, repo_root| + # Look for global repos (legacy, depth 1) and normal repos (depth 2) + IO.popen(%W(find #{repo_root} -mindepth 1 -maxdepth 2 -name *.git)) do |find| + find.each_line do |path| + path.chomp! + repo_with_namespace = path. + sub(repo_root, ''). + sub(%r{^/*}, ''). + chomp('.git'). + chomp('.wiki') + next if Project.find_with_namespace(repo_with_namespace) + new_path = path + move_suffix + puts path.inspect + ' -> ' + new_path.inspect + File.rename(path, new_path) + end end end end diff --git a/lib/tasks/gitlab/import.rake b/lib/tasks/gitlab/import.rake index 4753f00c26a..dbdd4e977e8 100644 --- a/lib/tasks/gitlab/import.rake +++ b/lib/tasks/gitlab/import.rake @@ -2,73 +2,73 @@ namespace :gitlab do namespace :import do # How to use: # - # 1. copy the bare repos under the repos_path (commonly /home/git/repositories) + # 1. copy the bare repos under the repository storage paths (commonly the default path is /home/git/repositories) # 2. run: bundle exec rake gitlab:import:repos RAILS_ENV=production # # Notes: # * The project owner will set to the first administator of the system # * Existing projects will be skipped # - desc "GitLab | Import bare repositories from gitlab_shell -> repos_path into GitLab project instance" + desc "GitLab | Import bare repositories from repositories -> storages into GitLab project instance" task repos: :environment do + Gitlab.config.repositories.storages.each do |name, git_base_path| + repos_to_import = Dir.glob(git_base_path + '/**/*.git') - git_base_path = Gitlab.config.gitlab_shell.repos_path - repos_to_import = Dir.glob(git_base_path + '/**/*.git') + repos_to_import.each do |repo_path| + # strip repo base path + repo_path[0..git_base_path.length] = '' - repos_to_import.each do |repo_path| - # strip repo base path - repo_path[0..git_base_path.length] = '' + path = repo_path.sub(/\.git$/, '') + group_name, name = File.split(path) + group_name = nil if group_name == '.' - path = repo_path.sub(/\.git$/, '') - group_name, name = File.split(path) - group_name = nil if group_name == '.' + puts "Processing #{repo_path}".color(:yellow) - puts "Processing #{repo_path}".color(:yellow) - - if path.end_with?('.wiki') - puts " * Skipping wiki repo" - next - end + if path.end_with?('.wiki') + puts " * Skipping wiki repo" + next + end - project = Project.find_with_namespace(path) + project = Project.find_with_namespace(path) - if project - puts " * #{project.name} (#{repo_path}) exists" - else - user = User.admins.reorder("id").first + if project + puts " * #{project.name} (#{repo_path}) exists" + else + user = User.admins.reorder("id").first - project_params = { - name: name, - path: name - } + project_params = { + name: name, + path: name + } - # find group namespace - if group_name - group = Namespace.find_by(path: group_name) - # create group namespace - unless group - group = Group.new(:name => group_name) - group.path = group_name - group.owner = user - if group.save - puts " * Created Group #{group.name} (#{group.id})".color(:green) - else - puts " * Failed trying to create group #{group.name}".color(:red) + # find group namespace + if group_name + group = Namespace.find_by(path: group_name) + # create group namespace + unless group + group = Group.new(:name => group_name) + group.path = group_name + group.owner = user + if group.save + puts " * Created Group #{group.name} (#{group.id})".color(:green) + else + puts " * Failed trying to create group #{group.name}".color(:red) + end end + # set project group + project_params[:namespace_id] = group.id end - # set project group - project_params[:namespace_id] = group.id - end - project = Projects::CreateService.new(user, project_params).execute + project = Projects::CreateService.new(user, project_params).execute - if project.persisted? - puts " * Created #{project.name} (#{repo_path})".color(:green) - project.update_repository_size - project.update_commit_count - else - puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) - puts " Errors: #{project.errors.messages}".color(:red) + if project.persisted? + puts " * Created #{project.name} (#{repo_path})".color(:green) + project.update_repository_size + project.update_commit_count + else + puts " * Failed trying to create #{project.name} (#{repo_path})".color(:red) + puts " Errors: #{project.errors.messages}".color(:red) + end end end end diff --git a/lib/tasks/gitlab/info.rake b/lib/tasks/gitlab/info.rake index 352b566df24..fe43d40e6d2 100644 --- a/lib/tasks/gitlab/info.rake +++ b/lib/tasks/gitlab/info.rake @@ -62,7 +62,10 @@ namespace :gitlab do puts "" puts "GitLab Shell".color(:yellow) puts "Version:\t#{gitlab_shell_version || "unknown".color(:red)}" - puts "Repositories:\t#{Gitlab.config.gitlab_shell.repos_path}" + puts "Repository storage paths:" + Gitlab.config.repositories.storages.each do |name, path| + puts "- #{name}: \t#{path}" + end puts "Hooks:\t\t#{Gitlab.config.gitlab_shell.hooks_path}" puts "Git:\t\t#{Gitlab.config.git.bin_path}" diff --git a/lib/tasks/gitlab/list_repos.rake b/lib/tasks/gitlab/list_repos.rake index c7596e7abcb..ffcc76e5498 100644 --- a/lib/tasks/gitlab/list_repos.rake +++ b/lib/tasks/gitlab/list_repos.rake @@ -9,7 +9,7 @@ namespace :gitlab do scope = scope.where('id IN (?) OR namespace_id in (?)', project_ids, namespace_ids) end scope.find_each do |project| - base = File.join(Gitlab.config.gitlab_shell.repos_path, project.path_with_namespace) + base = File.join(project.repository_storage_path, project.path_with_namespace) puts base + '.git' puts base + '.wiki.git' end diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index b1648a4602a..c85ebdf8619 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -12,7 +12,6 @@ namespace :gitlab do gitlab_url = Gitlab.config.gitlab.url # gitlab-shell requires a / at the end of the url gitlab_url += '/' unless gitlab_url.end_with?('/') - repos_path = Gitlab.config.gitlab_shell.repos_path target_dir = Gitlab.config.gitlab_shell.path # Clone if needed @@ -35,7 +34,6 @@ namespace :gitlab do user: user, gitlab_url: gitlab_url, http_settings: {self_signed_cert: false}.stringify_keys, - repos_path: repos_path, auth_file: File.join(home_dir, ".ssh", "authorized_keys"), redis: { bin: %x{which redis-cli}.chomp, @@ -58,10 +56,10 @@ namespace :gitlab do File.open("config.yml", "w+") {|f| f.puts config.to_yaml} # Launch installation process - system(*%W(bin/install)) + system(*%W(bin/install) + repository_storage_paths_args) # (Re)create hooks - system(*%W(bin/create-hooks)) + system(*%W(bin/create-hooks) + repository_storage_paths_args) end # Required for debian packaging with PKGR: Setup .ssh/environment with @@ -73,6 +71,8 @@ namespace :gitlab do File.open(File.join(home_dir, ".ssh", "environment"), "w+") do |f| f.puts "PATH=#{ENV['PATH']}" end + + Gitlab::Shell.new.generate_and_link_secret_token end desc "GitLab | Setup gitlab-shell" @@ -87,7 +87,8 @@ namespace :gitlab do if File.exists?(path_to_repo) print '-' else - if Gitlab::Shell.new.add_repository(project.path_with_namespace) + if Gitlab::Shell.new.add_repository(project.repository_storage_path, + project.path_with_namespace) print '.' else print 'F' @@ -138,4 +139,3 @@ namespace :gitlab do system(*%W(#{Gitlab.config.git.bin_path} reset --hard #{tag})) end end - diff --git a/lib/tasks/gitlab/task_helpers.rake b/lib/tasks/gitlab/task_helpers.rake index d0c019044b7..ab96b1d3593 100644 --- a/lib/tasks/gitlab/task_helpers.rake +++ b/lib/tasks/gitlab/task_helpers.rake @@ -125,10 +125,16 @@ namespace :gitlab do end def all_repos - IO.popen(%W(find #{Gitlab.config.gitlab_shell.repos_path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| - find.each_line do |path| - yield path.chomp + Gitlab.config.repositories.storages.each do |name, path| + IO.popen(%W(find #{path} -mindepth 2 -maxdepth 2 -type d -name *.git)) do |find| + find.each_line do |path| + yield path.chomp + end end end end + + def repository_storage_paths_args + Gitlab.config.repositories.storages.values + end end diff --git a/lib/tasks/test.rake b/lib/tasks/test.rake index c5666d49e61..21c0e5f1d41 100644 --- a/lib/tasks/test.rake +++ b/lib/tasks/test.rake @@ -6,8 +6,6 @@ task :test do end unless Rails.env.production? - require 'coveralls/rake/task' - Coveralls::RakeTask.new desc "GitLab | Run all tests on CI with simplecov" - task :test_ci => [:rubocop, :brakeman, 'teaspoon', :spinach, :spec, 'coveralls:push'] + task test_ci: [:rubocop, :brakeman, 'teaspoon', :spinach, :spec] end diff --git a/lib/uploaded_file.rb b/lib/uploaded_file.rb index d4291f012d3..41dee5fdc06 100644 --- a/lib/uploaded_file.rb +++ b/lib/uploaded_file.rb @@ -3,7 +3,6 @@ require "fileutils" # Taken from: Rack::Test::UploadedFile class UploadedFile - # The filename, *not* including the path, of the "uploaded" file attr_reader :original_filename |