diff options
Diffstat (limited to 'lib')
63 files changed, 1084 insertions, 632 deletions
diff --git a/lib/api/branches.rb b/lib/api/branches.rb index b615703df93..21a106387f0 100644 --- a/lib/api/branches.rb +++ b/lib/api/branches.rb @@ -6,124 +6,100 @@ module API before { authenticate! } before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project repository branches - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/repository/branches + desc 'Get a project repository branches' do + success Entities::RepoBranch + end get ":id/repository/branches" do branches = user_project.repository.branches.sort_by(&:name) present branches, with: Entities::RepoBranch, project: user_project end - # Get a single branch - # - # Parameters: - # id (required) - The ID of a project - # branch (required) - The name of the branch - # Example Request: - # GET /projects/:id/repository/branches/:branch - get ':id/repository/branches/:branch', requirements: { branch: /.+/ } do - @branch = user_project.repository.branches.find { |item| item.name == params[:branch] } - not_found!("Branch") unless @branch + desc 'Get a single branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + end + get ':id/repository/branches/:branch' do + branch = user_project.repository.find_branch(params[:branch]) + not_found!("Branch") unless branch - present @branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::RepoBranch, project: user_project end - # Protect a single branch - # # Note: The internal data model moved from `developers_can_{merge,push}` to `allowed_to_{merge,push}` # in `gitlab-org/gitlab-ce!5081`. The API interface has not been changed (to maintain compatibility), # but it works with the changed data model to infer `developers_can_merge` and `developers_can_push`. - # - # Parameters: - # id (required) - The ID of a project - # branch (required) - The name of the branch - # developers_can_push (optional) - Flag if developers can push to that branch - # developers_can_merge (optional) - Flag if developers can merge to that branch - # Example Request: - # PUT /projects/:id/repository/branches/:branch/protect - put ':id/repository/branches/:branch/protect', - requirements: { branch: /.+/ } do + desc 'Protect a single branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + optional :developers_can_push, type: Boolean, desc: 'Flag if developers can push to that branch' + optional :developers_can_merge, type: Boolean, desc: 'Flag if developers can merge to that branch' + end + put ':id/repository/branches/:branch/protect' do authorize_admin_project - @branch = user_project.repository.find_branch(params[:branch]) - not_found!('Branch') unless @branch - protected_branch = user_project.protected_branches.find_by(name: @branch.name) + branch = user_project.repository.find_branch(params[:branch]) + not_found!('Branch') unless branch - developers_can_merge = to_boolean(params[:developers_can_merge]) - developers_can_push = to_boolean(params[:developers_can_push]) + protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch_params = { - name: @branch.name + name: branch.name, + developers_can_push: params[:developers_can_push], + developers_can_merge: params[:developers_can_merge] } - # If `developers_can_merge` is switched off, _all_ `DEVELOPER` - # merge_access_levels need to be deleted. - if developers_can_merge == false - protected_branch.merge_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all - end + service_args = [user_project, current_user, protected_branch_params] - # If `developers_can_push` is switched off, _all_ `DEVELOPER` - # push_access_levels need to be deleted. - if developers_can_push == false - protected_branch.push_access_levels.where(access_level: Gitlab::Access::DEVELOPER).destroy_all - end + protected_branch = if protected_branch + ProtectedBranches::ApiUpdateService.new(*service_args).execute(protected_branch) + else + ProtectedBranches::ApiCreateService.new(*service_args).execute + end - protected_branch_params.merge!( - merge_access_levels_attributes: [{ - access_level: developers_can_merge ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - }], - push_access_levels_attributes: [{ - access_level: developers_can_push ? Gitlab::Access::DEVELOPER : Gitlab::Access::MASTER - }] - ) - - if protected_branch - service = ProtectedBranches::UpdateService.new(user_project, current_user, protected_branch_params) - service.execute(protected_branch) + if protected_branch.valid? + present branch, with: Entities::RepoBranch, project: user_project else - service = ProtectedBranches::CreateService.new(user_project, current_user, protected_branch_params) - service.execute + render_api_error!(protected_branch.errors.full_messages, 422) end - - present @branch, with: Entities::RepoBranch, project: user_project end - # Unprotect a single branch - # - # Parameters: - # id (required) - The ID of a project - # branch (required) - The name of the branch - # Example Request: - # PUT /projects/:id/repository/branches/:branch/unprotect - put ':id/repository/branches/:branch/unprotect', - requirements: { branch: /.+/ } do + desc 'Unprotect a single branch' do + success Entities::RepoBranch + end + params do + requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + end + put ':id/repository/branches/:branch/unprotect' do authorize_admin_project - @branch = user_project.repository.find_branch(params[:branch]) - not_found!("Branch") unless @branch - protected_branch = user_project.protected_branches.find_by(name: @branch.name) + branch = user_project.repository.find_branch(params[:branch]) + not_found!("Branch") unless branch + protected_branch = user_project.protected_branches.find_by(name: branch.name) protected_branch.destroy if protected_branch - present @branch, with: Entities::RepoBranch, project: user_project + present branch, with: Entities::RepoBranch, project: user_project end - # Create branch - # - # Parameters: - # id (required) - The ID of a project - # branch_name (required) - The name of the branch - # ref (required) - Create branch from commit sha or existing branch - # Example Request: - # POST /projects/:id/repository/branches + desc 'Create branch' do + success Entities::RepoBranch + end + params do + requires :branch_name, type: String, desc: 'The name of the branch' + requires :ref, type: String, desc: 'Create branch from commit sha or existing branch' + end post ":id/repository/branches" do authorize_push_project result = CreateBranchService.new(user_project, current_user). - execute(params[:branch_name], params[:ref]) + execute(params[:branch_name], params[:ref]) if result[:status] == :success present result[:branch], @@ -134,18 +110,15 @@ module API end end - # Delete branch - # - # Parameters: - # id (required) - The ID of a project - # branch (required) - The name of the branch - # Example Request: - # DELETE /projects/:id/repository/branches/:branch - delete ":id/repository/branches/:branch", - requirements: { branch: /.+/ } do + desc 'Delete a branch' + params do + requires :branch, type: String, regexp: /.+/, desc: 'The name of the branch' + end + delete ":id/repository/branches/:branch" do authorize_push_project + result = DeleteBranchService.new(user_project, current_user). - execute(params[:branch]) + execute(params[:branch]) if result[:status] == :success { diff --git a/lib/api/builds.rb b/lib/api/builds.rb index 52bdbcae5a8..67adca6605f 100644 --- a/lib/api/builds.rb +++ b/lib/api/builds.rb @@ -3,15 +3,32 @@ module API class Builds < Grape::API before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project builds - # - # Parameters: - # id (required) - The ID of a project - # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; - # if none provided showing all builds) - # Example Request: - # GET /projects/:id/builds + helpers do + params :optional_scope do + optional :scope, types: [String, Array[String]], desc: 'The scope of builds to show', + values: ['pending', 'running', 'failed', 'success', 'canceled'], + coerce_with: ->(scope) { + if scope.is_a?(String) + [scope] + elsif scope.is_a?(Hashie::Mash) + scope.values + else + ['unknown'] + end + } + end + end + + desc 'Get a project builds' do + success Entities::Build + end + params do + use :optional_scope + end get ':id/builds' do builds = user_project.builds.order('id DESC') builds = filter_builds(builds, params[:scope]) @@ -20,15 +37,13 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Get builds for a specific commit of a project - # - # Parameters: - # id (required) - The ID of a project - # sha (required) - The SHA id of a commit - # scope (optional) - The scope of builds to show (one or array of: pending, running, failed, success, canceled; - # if none provided showing all builds) - # Example Request: - # GET /projects/:id/repository/commits/:sha/builds + desc 'Get builds for a specific commit of a project' do + success Entities::Build + end + params do + requires :sha, type: String, desc: 'The SHA id of a commit' + use :optional_scope + end get ':id/repository/commits/:sha/builds' do authorize_read_builds! @@ -42,13 +57,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Get a specific build of a project - # - # Parameters: - # id (required) - The ID of a project - # build_id (required) - The ID of a build - # Example Request: - # GET /projects/:id/builds/:build_id + desc 'Get a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id' do authorize_read_builds! @@ -58,13 +72,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Download the artifacts file from build - # - # Parameters: - # id (required) - The ID of a build - # token (required) - The build authorization token - # Example Request: - # GET /projects/:id/builds/:build_id/artifacts + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.5' + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id/artifacts' do authorize_read_builds! @@ -73,14 +86,13 @@ module API present_artifacts!(build.artifacts_file) end - # Download the artifacts file from ref_name and job - # - # Parameters: - # id (required) - The ID of a project - # ref_name (required) - The ref from repository - # job (required) - The name for the build - # Example Request: - # GET /projects/:id/builds/artifacts/:ref_name/download?job=name + desc 'Download the artifacts file from build' do + detail 'This feature was introduced in GitLab 8.10' + end + params do + requires :ref_name, type: String, desc: 'The ref from repository' + requires :job, type: String, desc: 'The name for the build' + end get ':id/builds/artifacts/:ref_name/download', requirements: { ref_name: /.+/ } do authorize_read_builds! @@ -91,17 +103,13 @@ module API present_artifacts!(latest_build.artifacts_file) end - # Get a trace of a specific build of a project - # - # Parameters: - # id (required) - The ID of a project - # build_id (required) - The ID of a build - # Example Request: - # GET /projects/:id/build/:build_id/trace - # # TODO: We should use `present_file!` and leave this implementation for backward compatibility (when build trace # is saved in the DB instead of file). But before that, we need to consider how to replace the value of # `runners_token` with some mask (like `xxxxxx`) when sending trace file directly by workhorse. + desc 'Get a trace of a specific build of a project' + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end get ':id/builds/:build_id/trace' do authorize_read_builds! @@ -115,13 +123,12 @@ module API body trace end - # Cancel a specific build of a project - # - # parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example request: - # post /projects/:id/build/:build_id/cancel + desc 'Cancel a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/cancel' do authorize_update_builds! @@ -133,13 +140,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Retry a specific build of a project - # - # parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example request: - # post /projects/:id/build/:build_id/retry + desc 'Retry a specific build of a project' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/retry' do authorize_update_builds! @@ -152,13 +158,12 @@ module API user_can_download_artifacts: can?(current_user, :read_build, user_project) end - # Erase build (remove artifacts and build trace) - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - the id of a build - # example Request: - # post /projects/:id/build/:build_id/erase + desc 'Erase build (remove artifacts and build trace)' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/erase' do authorize_update_builds! @@ -170,13 +175,12 @@ module API user_can_download_artifacts: can?(current_user, :download_build_artifacts, user_project) end - # Keep the artifacts to prevent them from being deleted - # - # Parameters: - # id (required) - the id of a project - # build_id (required) - The ID of a build - # Example Request: - # POST /projects/:id/builds/:build_id/artifacts/keep + desc 'Keep the artifacts to prevent them from being deleted' do + success Entities::Build + end + params do + requires :build_id, type: Integer, desc: 'The ID of a build' + end post ':id/builds/:build_id/artifacts/keep' do authorize_update_builds! @@ -235,14 +239,6 @@ module API return builds if scope.nil? || scope.empty? available_statuses = ::CommitStatus::AVAILABLE_STATUSES - scope = - if scope.is_a?(String) - [scope] - elsif scope.is_a?(Hashie::Mash) - scope.values - else - ['unknown'] - end unknown = scope - available_statuses render_api_error!('Scope contains invalid value(s)', 400) unless unknown.empty? diff --git a/lib/api/commit_statuses.rb b/lib/api/commit_statuses.rb index f282a3b9cd6..f54d4f06627 100644 --- a/lib/api/commit_statuses.rb +++ b/lib/api/commit_statuses.rb @@ -67,9 +67,14 @@ module API pipeline = @project.ensure_pipeline(ref, commit.sha, current_user) status = GenericCommitStatus.running_or_pending.find_or_initialize_by( - project: @project, pipeline: pipeline, - user: current_user, name: name, ref: ref) - status.attributes = declared(params).slice(:target_url, :description) + project: @project, + pipeline: pipeline, + user: current_user, + name: name, + ref: ref, + target_url: params[:target_url], + description: params[:description] + ) begin case params[:state].to_s diff --git a/lib/api/commits.rb b/lib/api/commits.rb index 617a240318a..2f2cf769481 100644 --- a/lib/api/commits.rb +++ b/lib/api/commits.rb @@ -19,6 +19,7 @@ module API optional :until, type: String, desc: 'Only commits before or in this date will be returned' optional :page, type: Integer, default: 0, desc: 'The page for pagination' optional :per_page, type: Integer, default: 20, desc: 'The number of results per page' + optional :path, type: String, desc: 'The file path' end get ":id/repository/commits" do # TODO remove the next line for 9.0, use DateTime type in the params block @@ -28,6 +29,7 @@ module API offset = params[:page] * params[:per_page] commits = user_project.repository.commits(ref, + path: params[:path], limit: params[:per_page], offset: offset, after: params[:since], diff --git a/lib/api/deploy_keys.rb b/lib/api/deploy_keys.rb index 825e05fbae3..425df2c176a 100644 --- a/lib/api/deploy_keys.rb +++ b/lib/api/deploy_keys.rb @@ -49,18 +49,23 @@ module API attrs = attributes_for_keys [:title, :key] attrs[:key].strip! if attrs[:key] + # Check for an existing key joined to this project key = user_project.deploy_keys.find_by(key: attrs[:key]) - present key, with: Entities::SSHKey if key + if key + present key, with: Entities::SSHKey + break + end # Check for available deploy keys in other projects key = current_user.accessible_deploy_keys.find_by(key: attrs[:key]) if key user_project.deploy_keys << key present key, with: Entities::SSHKey + break end + # Create a new deploy key key = DeployKey.new attrs - if key.valid? && user_project.deploy_keys << key present key, with: Entities::SSHKey else diff --git a/lib/api/entities.rb b/lib/api/entities.rb index feaa0c213bf..01e31f6f7d1 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -43,14 +43,13 @@ module API end class Hook < Grape::Entity - expose :id, :url, :created_at + expose :id, :url, :created_at, :push_events, :tag_push_events + expose :enable_ssl_verification end class ProjectHook < Hook - expose :project_id, :push_events - expose :issues_events, :merge_requests_events, :tag_push_events + expose :project_id, :issues_events, :merge_requests_events expose :note_events, :build_events, :pipeline_events, :wiki_page_events - expose :enable_ssl_verification end class BasicProjectDetails < Grape::Entity @@ -101,6 +100,7 @@ module API end expose :only_allow_merge_if_build_succeeds expose :request_access_enabled + expose :only_allow_merge_if_all_discussions_are_resolved end class Member < UserBasic @@ -138,7 +138,7 @@ module API expose :name expose :commit do |repo_branch, options| - options[:project].repository.commit(repo_branch.target) + options[:project].repository.commit(repo_branch.dereferenced_target) end expose :protected do |repo_branch, options| @@ -510,6 +510,7 @@ module API expose :after_sign_out_path expose :container_registry_token_expire_delay expose :repository_storage + expose :repository_storages expose :koding_enabled expose :koding_url end @@ -523,7 +524,7 @@ module API expose :name, :message expose :commit do |repo_tag, options| - options[:project].repository.commit(repo_tag.target) + options[:project].repository.commit(repo_tag.dereferenced_target) end expose :release, using: Entities::Release do |repo_tag, options| diff --git a/lib/api/groups.rb b/lib/api/groups.rb index bfb89475025..a13e353b7f5 100644 --- a/lib/api/groups.rb +++ b/lib/api/groups.rb @@ -8,11 +8,14 @@ module API # # Parameters: # skip_groups (optional) - Array of group ids to exclude from list + # all_available (optional, boolean) - Show all group that you have access to # Example Request: # GET /groups get do @groups = if current_user.admin Group.all + elsif params[:all_available] + GroupsFinder.new.execute(current_user) else current_user.groups end diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 45120898b76..3c9d7b1aaef 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -1,17 +1,12 @@ module API module Helpers + include Gitlab::Utils + PRIVATE_TOKEN_HEADER = "HTTP_PRIVATE_TOKEN" PRIVATE_TOKEN_PARAM = :private_token SUDO_HEADER = "HTTP_SUDO" SUDO_PARAM = :sudo - def to_boolean(value) - return true if value =~ /^(true|t|yes|y|1|on)$/i - return false if value =~ /^(false|f|no|n|0|off)$/i - - nil - end - def private_token params[PRIVATE_TOKEN_PARAM] || env[PRIVATE_TOKEN_HEADER] end diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 9a5d1ece070..ccf181402f9 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -17,15 +17,20 @@ module API # helpers do + def project_path + @project_path ||= begin + project_path = params[:project].sub(/\.git\z/, '') + Repository.remove_storage_from_path(project_path) + end + end + def wiki? - @wiki ||= params[:project].end_with?('.wiki') && - !Project.find_with_namespace(params[:project]) + @wiki ||= project_path.end_with?('.wiki') && + !Project.find_with_namespace(project_path) end def project @project ||= begin - project_path = params[:project] - # Check for *.wiki repositories. # Strip out the .wiki from the pathname before finding the # project. This applies the correct project permissions to diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 40d7f4a5151..238cea00fba 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -3,37 +3,32 @@ module API class Labels < Grape::API before { authenticate! } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get all labels of the project - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/labels + desc 'Get all labels of the project' do + success Entities::Label + end get ':id/labels' do present available_labels, with: Entities::Label, current_user: current_user end - # Creates a new label - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be created - # color (required) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) - # description (optional) - The description of label to be created - # Example Request: - # POST /projects/:id/labels + desc 'Create a new label' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be created' + requires :color, type: String, desc: "The color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :description, type: String, desc: 'The description of label to be created' + end post ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name, :color] - - attrs = attributes_for_keys [:name, :color, :description] - label = available_labels.find_by(title: attrs[:name]) + label = available_labels.find_by(title: params[:name]) conflict!('Label already exists') if label - label = user_project.labels.create(attrs) + label = user_project.labels.create(declared(params, include_parent_namespaces: false).to_h) if label.valid? present label, with: Entities::Label, current_user: current_user @@ -42,54 +37,44 @@ module API end end - # Deletes an existing label - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # - # Example Request: - # DELETE /projects/:id/labels + desc 'Delete an existing label' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be deleted' + end delete ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name] label = user_project.labels.find_by(title: params[:name]) not_found!('Label') unless label - label.destroy + present label.destroy, with: Entities::Label, current_user: current_user end - # Updates an existing label. At least one optional parameter is required. - # - # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # new_name (optional) - The new name of the label - # color (optional) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) - # description (optional) - The description of label to be created - # Example Request: - # PUT /projects/:id/labels + desc 'Update an existing label. At least one optional parameter is required.' do + success Entities::Label + end + params do + requires :name, type: String, desc: 'The name of the label to be updated' + optional :new_name, type: String, desc: 'The new name of the label' + optional :color, type: String, desc: "The new color of the label given in 6-digit hex notation with leading '#' sign (e.g. #FFAABB)" + optional :description, type: String, desc: 'The new description of label' + at_least_one_of :new_name, :color, :description + end put ':id/labels' do authorize! :admin_label, user_project - required_attributes! [:name] label = user_project.labels.find_by(title: params[:name]) not_found!('Label not found') unless label - attrs = attributes_for_keys [:new_name, :color, :description] - - if attrs.empty? - render_api_error!('Required parameters "new_name" or "color" ' \ - 'missing', - 400) - end - + update_params = declared(params, + include_parent_namespaces: false, + include_missing: false).to_h # Rename new name to the actual label attribute name - attrs[:name] = attrs.delete(:new_name) if attrs.key?(:new_name) + update_params['name'] = update_params.delete('new_name') if update_params.key?('new_name') - if label.update(attrs) + if label.update(update_params) present label, with: Entities::Label, current_user: current_user else render_validation_error!(label) diff --git a/lib/api/project_hooks.rb b/lib/api/project_hooks.rb index 14f5be3b5f6..eef343c2ac6 100644 --- a/lib/api/project_hooks.rb +++ b/lib/api/project_hooks.rb @@ -1,112 +1,99 @@ module API # Projects API class ProjectHooks < Grape::API + helpers do + params :project_hook_properties do + requires :url, type: String, desc: "The URL to send the request to" + optional :push_events, type: Boolean, desc: "Trigger hook on push events" + optional :issues_events, type: Boolean, desc: "Trigger hook on issues events" + optional :merge_requests_events, type: Boolean, desc: "Trigger hook on merge request events" + optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :note_events, type: Boolean, desc: "Trigger hook on note(comment) events" + optional :build_events, type: Boolean, desc: "Trigger hook on build events" + optional :pipeline_events, type: Boolean, desc: "Trigger hook on pipeline events" + optional :wiki_events, type: Boolean, desc: "Trigger hook on wiki events" + optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" + optional :token, type: String, desc: "Secret token to validate received payloads; this will not be returned in the response" + end + end + before { authenticate! } before { authorize_admin_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get project hooks - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/hooks + desc 'Get project hooks' do + success Entities::ProjectHook + end get ":id/hooks" do - @hooks = paginate user_project.hooks - present @hooks, with: Entities::ProjectHook + hooks = paginate user_project.hooks + + present hooks, with: Entities::ProjectHook end - # Get a project hook - # - # Parameters: - # id (required) - The ID of a project - # hook_id (required) - The ID of a project hook - # Example Request: - # GET /projects/:id/hooks/:hook_id + desc 'Get a project hook' do + success Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: 'The ID of a project hook' + end get ":id/hooks/:hook_id" do - @hook = user_project.hooks.find(params[:hook_id]) - present @hook, with: Entities::ProjectHook + hook = user_project.hooks.find(params[:hook_id]) + present hook, with: Entities::ProjectHook end - # Add hook to project - # - # Parameters: - # id (required) - The ID of a project - # url (required) - The hook URL - # Example Request: - # POST /projects/:id/hooks + desc 'Add hook to project' do + success Entities::ProjectHook + end + params do + use :project_hook_properties + end post ":id/hooks" do - required_attributes! [:url] - attrs = attributes_for_keys [ - :url, - :push_events, - :issues_events, - :merge_requests_events, - :tag_push_events, - :note_events, - :build_events, - :pipeline_events, - :wiki_page_events, - :enable_ssl_verification - ] - @hook = user_project.hooks.new(attrs) + new_hook_params = declared(params, include_missing: false, include_parent_namespaces: false).to_h + hook = user_project.hooks.new(new_hook_params) - if @hook.save - present @hook, with: Entities::ProjectHook + if hook.save + present hook, with: Entities::ProjectHook else - if @hook.errors[:url].present? - error!("Invalid url given", 422) - end - not_found!("Project hook #{@hook.errors.messages}") + error!("Invalid url given", 422) if hook.errors[:url].present? + + not_found!("Project hook #{hook.errors.messages}") end end - # Update an existing project hook - # - # Parameters: - # id (required) - The ID of a project - # hook_id (required) - The ID of a project hook - # url (required) - The hook URL - # Example Request: - # PUT /projects/:id/hooks/:hook_id + desc 'Update an existing project hook' do + success Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: "The ID of the hook to update" + use :project_hook_properties + end put ":id/hooks/:hook_id" do - @hook = user_project.hooks.find(params[:hook_id]) - required_attributes! [:url] - attrs = attributes_for_keys [ - :url, - :push_events, - :issues_events, - :merge_requests_events, - :tag_push_events, - :note_events, - :build_events, - :pipeline_events, - :wiki_page_events, - :enable_ssl_verification - ] + hook = user_project.hooks.find(params[:hook_id]) + + new_params = declared(params, include_missing: false, include_parent_namespaces: false).to_h + new_params.delete('hook_id') - if @hook.update_attributes attrs - present @hook, with: Entities::ProjectHook + if hook.update_attributes(new_params) + present hook, with: Entities::ProjectHook else - if @hook.errors[:url].present? - error!("Invalid url given", 422) - end - not_found!("Project hook #{@hook.errors.messages}") + error!("Invalid url given", 422) if hook.errors[:url].present? + + not_found!("Project hook #{hook.errors.messages}") end end - # Deletes project hook. This is an idempotent function. - # - # Parameters: - # id (required) - The ID of a project - # hook_id (required) - The ID of hook to delete - # Example Request: - # DELETE /projects/:id/hooks/:hook_id + desc 'Deletes project hook' do + success Entities::ProjectHook + end + params do + requires :hook_id, type: Integer, desc: 'The ID of the hook to delete' + end delete ":id/hooks/:hook_id" do - required_attributes! [:hook_id] - begin - @hook = user_project.hooks.destroy(params[:hook_id]) + present user_project.hooks.destroy(params[:hook_id]), with: Entities::ProjectHook rescue # ProjectHook can raise Error if hook_id not found not_found!("Error deleting hook #{params[:hook_id]}") diff --git a/lib/api/projects.rb b/lib/api/projects.rb index da16e24d7ea..6b856128c2e 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -139,7 +139,8 @@ module API :shared_runners_enabled, :snippets_enabled, :visibility_level, - :wiki_enabled] + :wiki_enabled, + :only_allow_merge_if_all_discussions_are_resolved] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(current_user, attrs).execute if @project.saved? @@ -193,7 +194,8 @@ module API :shared_runners_enabled, :snippets_enabled, :visibility_level, - :wiki_enabled] + :wiki_enabled, + :only_allow_merge_if_all_discussions_are_resolved] attrs = map_public_to_visibility_level(attrs) @project = ::Projects::CreateService.new(user, attrs).execute if @project.saved? @@ -275,7 +277,8 @@ module API :shared_runners_enabled, :snippets_enabled, :visibility_level, - :wiki_enabled] + :wiki_enabled, + :only_allow_merge_if_all_discussions_are_resolved] attrs = map_public_to_visibility_level(attrs) authorize_admin_project authorize! :rename_project, user_project if attrs[:name].present? diff --git a/lib/api/settings.rb b/lib/api/settings.rb index c885fcd7ea3..c4cb1c7924a 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -17,12 +17,12 @@ module API present current_settings, with: Entities::ApplicationSetting end - # Modify applicaiton settings + # Modify application settings # # Example Request: # PUT /application/settings put "application/settings" do - attributes = current_settings.attributes.keys - ["id"] + attributes = ["repository_storage"] + current_settings.attributes.keys - ["id"] attrs = attributes_for_keys(attributes) if current_settings.update_attributes(attrs) diff --git a/lib/api/system_hooks.rb b/lib/api/system_hooks.rb index 794e34874f4..32f731c5652 100644 --- a/lib/api/system_hooks.rb +++ b/lib/api/system_hooks.rb @@ -12,6 +12,7 @@ module API end get do hooks = SystemHook.all + present hooks, with: Entities::Hook end @@ -19,10 +20,14 @@ module API success Entities::Hook end params do - requires :url, type: String, desc: 'The URL for the system hook' + requires :url, type: String, desc: "The URL to send the request to" + optional :token, type: String, desc: 'The token used to validate payloads' + optional :push_events, type: Boolean, desc: "Trigger hook on push events" + optional :tag_push_events, type: Boolean, desc: "Trigger hook on tag push events" + optional :enable_ssl_verification, type: Boolean, desc: "Do SSL verification when triggering the hook" end post do - hook = SystemHook.new declared(params).to_h + hook = SystemHook.new declared(params, include_missing: false).to_h if hook.save present hook, with: Entities::Hook diff --git a/lib/api/tags.rb b/lib/api/tags.rb index 7b675e05fbb..bf2a199ce21 100644 --- a/lib/api/tags.rb +++ b/lib/api/tags.rb @@ -4,25 +4,24 @@ module API before { authenticate! } before { authorize! :download_code, user_project } + params do + requires :id, type: String, desc: 'The ID of a project' + end resource :projects do - # Get a project repository tags - # - # Parameters: - # id (required) - The ID of a project - # Example Request: - # GET /projects/:id/repository/tags + desc 'Get a project repository tags' do + success Entities::RepoTag + end get ":id/repository/tags" do present user_project.repository.tags.sort_by(&:name).reverse, with: Entities::RepoTag, project: user_project end - # Get a single repository tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # Example Request: - # GET /projects/:id/repository/tags/:tag_name + desc 'Get a single repository tag' do + success Entities::RepoTag + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + end get ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do tag = user_project.repository.find_tag(params[:tag_name]) not_found!('Tag') unless tag @@ -30,20 +29,21 @@ module API present tag, with: Entities::RepoTag, project: user_project end - # Create tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # ref (required) - Create tag from commit sha or branch - # message (optional) - Specifying a message creates an annotated tag. - # Example Request: - # POST /projects/:id/repository/tags + desc 'Create a new repository tag' do + success Entities::RepoTag + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + requires :ref, type: String, desc: 'The commit sha or branch name' + optional :message, type: String, desc: 'Specifying a message creates an annotated tag' + optional :release_description, type: String, desc: 'Specifying release notes stored in the GitLab database' + end post ':id/repository/tags' do authorize_push_project - message = params[:message] || nil + create_params = declared(params) + result = CreateTagService.new(user_project, current_user). - execute(params[:tag_name], params[:ref], message, params[:release_description]) + execute(create_params[:tag_name], create_params[:ref], create_params[:message], create_params[:release_description]) if result[:status] == :success present result[:tag], @@ -54,15 +54,13 @@ module API end end - # Delete tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # Example Request: - # DELETE /projects/:id/repository/tags/:tag + desc 'Delete a repository tag' + params do + requires :tag_name, type: String, desc: 'The name of the tag' + end delete ":id/repository/tags/:tag_name", requirements: { tag_name: /.+/ } do authorize_push_project + result = DeleteTagService.new(user_project, current_user). execute(params[:tag_name]) @@ -75,17 +73,16 @@ module API end end - # Add release notes to tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # description (required) - Release notes with markdown support - # Example Request: - # POST /projects/:id/repository/tags/:tag_name/release + desc 'Add a release note to a tag' do + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + requires :description, type: String, desc: 'Release notes with markdown support' + end post ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project - required_attributes! [:description] + result = CreateReleaseService.new(user_project, current_user). execute(params[:tag_name], params[:description]) @@ -96,17 +93,16 @@ module API end end - # Updates a release notes of a tag - # - # Parameters: - # id (required) - The ID of a project - # tag_name (required) - The name of the tag - # description (required) - Release notes with markdown support - # Example Request: - # PUT /projects/:id/repository/tags/:tag_name/release + desc "Update a tag's release note" do + success Entities::Release + end + params do + requires :tag_name, type: String, desc: 'The name of the tag' + requires :description, type: String, desc: 'Release notes with markdown support' + end put ':id/repository/tags/:tag_name/release', requirements: { tag_name: /.+/ } do authorize_push_project - required_attributes! [:description] + result = UpdateReleaseService.new(user_project, current_user). execute(params[:tag_name], params[:description]) diff --git a/lib/api/users.rb b/lib/api/users.rb index e868f628404..c28e07a76b7 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -333,11 +333,11 @@ module API user = User.find_by(id: declared(params).id) not_found!('User') unless user - events = user.recent_events. + events = user.events. merge(ProjectsFinder.new.execute(current_user)). references(:project). with_associations. - page(params[:page]) + recent present paginate(events), with: Entities::Event end diff --git a/lib/backup/repository.rb b/lib/backup/repository.rb index 9fcd9a3f999..d746070913d 100644 --- a/lib/backup/repository.rb +++ b/lib/backup/repository.rb @@ -2,11 +2,14 @@ require 'yaml' module Backup class Repository + def dump prepare Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " + path_to_project_repo = path_to_repo(project) + path_to_project_bundle = path_to_bundle(project) # Create namespace dir if missing FileUtils.mkdir_p(File.join(backup_repos_path, project.namespace.path)) if project.namespace @@ -14,8 +17,22 @@ module Backup if project.empty_repo? $progress.puts "[SKIPPED]".color(:cyan) else - cmd = %W(tar -cf #{path_to_bundle(project)} -C #{path_to_repo(project)} .) + in_path(path_to_project_repo) do |dir| + FileUtils.mkdir_p(path_to_tars(project)) + cmd = %W(tar -cf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) + output, status = Gitlab::Popen.popen(cmd) + + unless status.zero? + puts "[FAILED]".color(:red) + puts "failed: #{cmd.join(' ')}" + puts output + abort 'Backup failed' + end + end + + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_project_repo} bundle create #{path_to_project_bundle} --all) output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts "[DONE]".color(:green) else @@ -27,19 +44,22 @@ module Backup end wiki = ProjectWiki.new(project) + path_to_wiki_repo = path_to_repo(wiki) + path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_repo(wiki)) + if File.exist?(path_to_wiki_repo) $progress.print " * #{wiki.path_with_namespace} ... " if wiki.repository.empty? $progress.puts " [SKIPPED]".color(:cyan) else - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_repo(wiki)} bundle create #{path_to_bundle(wiki)} --all) + cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{path_to_wiki_repo} bundle create #{path_to_wiki_bundle} --all) output, status = Gitlab::Popen.popen(cmd) if status.zero? $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" + puts output abort 'Backup failed' end end @@ -60,40 +80,59 @@ module Backup Project.find_each(batch_size: 1000) do |project| $progress.print " * #{project.path_with_namespace} ... " + path_to_project_repo = path_to_repo(project) + path_to_project_bundle = path_to_bundle(project) project.ensure_dir_exist - if File.exist?(path_to_bundle(project)) - FileUtils.mkdir_p(path_to_repo(project)) - cmd = %W(tar -xf #{path_to_bundle(project)} -C #{path_to_repo(project)}) + if File.exists?(path_to_project_bundle) + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_project_bundle} #{path_to_project_repo}) else - cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_repo(project)}) + cmd = %W(#{Gitlab.config.git.bin_path} init --bare #{path_to_project_repo}) end - if system(*cmd, silent) + output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts "[DONE]".color(:green) else puts "[FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" + puts output abort 'Restore failed' end + in_path(path_to_tars(project)) do |dir| + cmd = %W(tar -xf #{path_to_tars(project, dir)} -C #{path_to_project_repo} #{dir}) + + output, status = Gitlab::Popen.popen(cmd) + unless status.zero? + puts "[FAILED]".color(:red) + puts "failed: #{cmd.join(' ')}" + puts output + abort 'Restore failed' + end + end + wiki = ProjectWiki.new(project) + path_to_wiki_repo = path_to_repo(wiki) + path_to_wiki_bundle = path_to_bundle(wiki) - if File.exist?(path_to_bundle(wiki)) + if File.exist?(path_to_wiki_bundle) $progress.print " * #{wiki.path_with_namespace} ... " # If a wiki bundle exists, first remove the empty repo # that was initialized with ProjectWiki.new() and then # try to restore with 'git clone --bare'. - FileUtils.rm_rf(path_to_repo(wiki)) - cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_bundle(wiki)} #{path_to_repo(wiki)}) + FileUtils.rm_rf(path_to_wiki_repo) + cmd = %W(#{Gitlab.config.git.bin_path} clone --bare #{path_to_wiki_bundle} #{path_to_wiki_repo}) - if system(*cmd, silent) + output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd.join(' ')}" + puts output abort 'Restore failed' end end @@ -101,13 +140,15 @@ module Backup $progress.print 'Put GitLab hooks in repositories dirs'.color(:yellow) cmd = %W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args - if system(*cmd) + + output, status = Gitlab::Popen.popen(cmd) + if status.zero? $progress.puts " [DONE]".color(:green) else puts " [FAILED]".color(:red) puts "failed: #{cmd}" + puts output end - end protected @@ -117,11 +158,30 @@ module Backup end def path_to_bundle(project) - File.join(backup_repos_path, project.path_with_namespace + ".bundle") + File.join(backup_repos_path, project.path_with_namespace + '.bundle') + end + + def path_to_tars(project, dir = nil) + path = File.join(backup_repos_path, project.path_with_namespace) + + if dir + File.join(path, "#{dir}.tar") + else + path + end end def backup_repos_path - File.join(Gitlab.config.backup.path, "repositories") + File.join(Gitlab.config.backup.path, 'repositories') + end + + def in_path(path) + return unless Dir.exist?(path) + + dir_entries = Dir.entries(path) + %w[annex custom_hooks].each do |entry| + yield(entry) if dir_entries.include?(entry) + end end def prepare diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index cb213a76a05..3740d4fb4cd 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -102,10 +102,10 @@ module Banzai end elsif element_node?(node) - yield_valid_link(node) do |link, text| + yield_valid_link(node) do |link, inner_html| if ref_pattern && link =~ /\A#{ref_pattern}\z/ replace_link_node_with_href(node, link) do - object_link_filter(link, ref_pattern, link_text: text) + object_link_filter(link, ref_pattern, link_content: inner_html) end next @@ -113,9 +113,9 @@ module Banzai next unless link_pattern - if link == text && text =~ /\A#{link_pattern}/ + if link == inner_html && inner_html =~ /\A#{link_pattern}/ replace_link_node_with_text(node, link) do - object_link_filter(text, link_pattern) + object_link_filter(inner_html, link_pattern) end next @@ -123,7 +123,7 @@ module Banzai if link =~ /\A#{link_pattern}\z/ replace_link_node_with_href(node, link) do - object_link_filter(link, link_pattern, link_text: text) + object_link_filter(link, link_pattern, link_content: inner_html) end next @@ -140,11 +140,11 @@ module Banzai # # text - String text to replace references in. # pattern - Reference pattern to match against. - # link_text - Original content of the link being replaced. + # link_content - Original content of the link being replaced. # # Returns a String with references replaced with links. All links # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. - def object_link_filter(text, pattern, link_text: nil) + def object_link_filter(text, pattern, link_content: nil) references_in(text, pattern) do |match, id, project_ref, matches| project = project_from_ref_cached(project_ref) @@ -152,7 +152,7 @@ module Banzai title = object_link_title(object) klass = reference_class(object_sym) - data = data_attributes_for(link_text || match, project, object) + data = data_attributes_for(link_content || match, project, object) if matches.names.include?("url") && matches[:url] url = matches[:url] @@ -160,11 +160,11 @@ module Banzai url = url_for_object_cached(object, project) end - text = link_text || object_link_text(object, matches) + content = link_content || object_link_text(object, matches) %(<a href="#{url}" #{data} title="#{escape_once(title)}" - class="#{klass}">#{escape_once(text)}</a>) + class="#{klass}">#{content}</a>) else match end diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index 0d20be557a0..dce4de3ceaf 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -37,10 +37,10 @@ module Banzai end elsif element_node?(node) - yield_valid_link(node) do |link, text| + yield_valid_link(node) do |link, inner_html| if link =~ ref_start_pattern replace_link_node_with_href(node, link) do - issue_link_filter(link, link_text: text) + issue_link_filter(link, link_content: inner_html) end end end @@ -54,10 +54,11 @@ module Banzai # issue's details page. # # text - String text to replace references in. + # link_content - Original content of the link being replaced. # # Returns a String with `JIRA-123` references replaced with links. All # links have `gfm` and `gfm-issue` class names attached for styling. - def issue_link_filter(text, link_text: nil) + def issue_link_filter(text, link_content: nil) project = context[:project] self.class.references_in(text, issue_reference_pattern) do |match, id| @@ -69,11 +70,11 @@ module Banzai klass = reference_class(:issue) data = data_attribute(project: project.id, external_issue: id) - text = link_text || match + content = link_content || match %(<a href="#{url}" #{data} title="#{escape_once(title)}" - class="#{klass}">#{escape_once(text)}</a>) + class="#{klass}">#{content}</a>) end end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index c24831e68ee..9f9a96cdc65 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -39,7 +39,7 @@ module Banzai end def find_labels(project) - LabelsFinder.new(nil, project_id: project.id).execute(authorized_only: false) + LabelsFinder.new(nil, project_id: project.id).execute(skip_authorization: true) end # Parameters to pass to `Label.find_by` based on the given arguments diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 2d221290f7e..84bfeac8041 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -85,14 +85,14 @@ module Banzai @nodes ||= each_node.to_a end - # Yields the link's URL and text whenever the node is a valid <a> tag. + # Yields the link's URL and inner HTML whenever the node is a valid <a> tag. def yield_valid_link(node) link = CGI.unescape(node.attr('href').to_s) - text = node.text + inner_html = node.inner_html return unless link.force_encoding('UTF-8').valid_encoding? - yield link, text + yield link, inner_html end def replace_text_when_pattern_matches(node, pattern) diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 4fa8d05481f..f09d78be0ce 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -52,8 +52,8 @@ module Banzai relative_url_root, context[:project].path_with_namespace, uri_type(file_path), - ref, - file_path + Addressable::URI.escape(ref), + Addressable::URI.escape(file_path) ].compact.join('/').squeeze('/').chomp('/') uri diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index c6302b586d3..f842b1fb779 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -35,10 +35,10 @@ module Banzai user_link_filter(content) end elsif element_node?(node) - yield_valid_link(node) do |link, text| + yield_valid_link(node) do |link, inner_html| if link =~ ref_pattern_start replace_link_node_with_href(node, link) do - user_link_filter(link, link_text: text) + user_link_filter(link, link_content: inner_html) end end end @@ -52,15 +52,16 @@ module Banzai # user's profile page. # # text - String text to replace references in. + # link_content - Original content of the link being replaced. # # Returns a String with `@user` references replaced with links. All links # have `gfm` and `gfm-project_member` class names attached for styling. - def user_link_filter(text, link_text: nil) + def user_link_filter(text, link_content: nil) self.class.references_in(text) do |match, username| if username == 'all' - link_to_all(link_text: link_text) + link_to_all(link_content: link_content) elsif namespace = namespaces[username] - link_to_namespace(namespace, link_text: link_text) || match + link_to_namespace(namespace, link_content: link_content) || match else match end @@ -102,49 +103,49 @@ module Banzai reference_class(:project_member) end - def link_to_all(link_text: nil) + def link_to_all(link_content: nil) project = context[:project] author = context[:author] if author && !project.team.member?(author) - link_text + link_content else url = urls.namespace_project_url(project.namespace, project, only_path: context[:only_path]) data = data_attribute(project: project.id, author: author.try(:id)) - text = link_text || User.reference_prefix + 'all' + content = link_content || User.reference_prefix + 'all' - link_tag(url, data, text, 'All Project and Group Members') + link_tag(url, data, content, 'All Project and Group Members') end end - def link_to_namespace(namespace, link_text: nil) + def link_to_namespace(namespace, link_content: nil) if namespace.is_a?(Group) - link_to_group(namespace.path, namespace, link_text: link_text) + link_to_group(namespace.path, namespace, link_content: link_content) else - link_to_user(namespace.path, namespace, link_text: link_text) + link_to_user(namespace.path, namespace, link_content: link_content) end end - def link_to_group(group, namespace, link_text: nil) + def link_to_group(group, namespace, link_content: nil) url = urls.group_url(group, only_path: context[:only_path]) data = data_attribute(group: namespace.id) - text = link_text || Group.reference_prefix + group + content = link_content || Group.reference_prefix + group - link_tag(url, data, text, namespace.name) + link_tag(url, data, content, namespace.name) end - def link_to_user(user, namespace, link_text: nil) + def link_to_user(user, namespace, link_content: nil) url = urls.user_url(user, only_path: context[:only_path]) data = data_attribute(user: namespace.owner_id) - text = link_text || User.reference_prefix + user + content = link_content || User.reference_prefix + user - link_tag(url, data, text, namespace.owner_name) + link_tag(url, data, content, namespace.owner_name) end - def link_tag(url, data, text, title) - %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{escape_once(text)}</a>) + def link_tag(url, data, link_content, title) + %(<a href="#{url}" #{data} class="#{link_class}" title="#{escape_once(title)}">#{link_content}</a>) end end end diff --git a/lib/banzai/redactor.rb b/lib/banzai/redactor.rb index 0df3a72d1c4..de3ebe72720 100644 --- a/lib/banzai/redactor.rb +++ b/lib/banzai/redactor.rb @@ -41,10 +41,10 @@ module Banzai next if visible.include?(node) doc_data[:visible_reference_count] -= 1 - # The reference should be replaced by the original text, - # which is not always the same as the rendered text. - text = node.attr('data-original') || node.text - node.replace(text) + # The reference should be replaced by the original link's content, + # which is not always the same as the rendered one. + content = node.attr('data-original') || node.inner_html + node.replace(content) end end diff --git a/lib/constraints/namespace_url_constrainer.rb b/lib/constraints/namespace_url_constrainer.rb index 23920193743..91b70143f11 100644 --- a/lib/constraints/namespace_url_constrainer.rb +++ b/lib/constraints/namespace_url_constrainer.rb @@ -1,6 +1,9 @@ class NamespaceUrlConstrainer def matches?(request) - id = request.path.sub(/\A\/+/, '').split('/').first.sub(/.atom\z/, '') + id = request.path + id = id.sub(/\A#{relative_url_root}/, '') if relative_url_root + id = id.sub(/\A\/+/, '').split('/').first + id = id.sub(/.atom\z/, '') if id if id =~ Gitlab::Regex.namespace_regex find_resource(id) @@ -10,4 +13,12 @@ class NamespaceUrlConstrainer def find_resource(id) Namespace.find_by_path(id) end + + private + + def relative_url_root + if defined?(Gitlab::Application.config.relative_url_root) + Gitlab::Application.config.relative_url_root + end + end end diff --git a/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb new file mode 100644 index 00000000000..7cb4bccb23c --- /dev/null +++ b/lib/generators/rails/post_deployment_migration/post_deployment_migration_generator.rb @@ -0,0 +1,15 @@ +require 'rails/generators' + +module Rails + class PostDeploymentMigrationGenerator < Rails::Generators::NamedBase + def create_migration_file + timestamp = Time.now.strftime('%Y%m%d%H%I%S') + + template "migration.rb", "db/post_migrate/#{timestamp}_#{file_name}.rb" + end + + def migration_class_name + file_name.camelize + end + end +end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index 4f81863da35..d76aa38f741 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -83,7 +83,7 @@ module Gitlab tag = repository.find_tag(tag_name) if tag - commit = repository.commit(tag.target) + commit = repository.commit(tag.dereferenced_target) commit.try(:sha) end else diff --git a/lib/gitlab/ee_compat_check.rb b/lib/gitlab/ee_compat_check.rb new file mode 100644 index 00000000000..b1a6d5fe0f6 --- /dev/null +++ b/lib/gitlab/ee_compat_check.rb @@ -0,0 +1,261 @@ +# rubocop: disable Rails/Output +module Gitlab + # Checks if a set of migrations requires downtime or not. + class EeCompatCheck + EE_REPO = 'https://gitlab.com/gitlab-org/gitlab-ee.git'.freeze + + attr_reader :ce_branch, :check_dir, :ce_repo + + def initialize(branch:, check_dir:, ce_repo: nil) + @ce_branch = branch + @check_dir = check_dir + @ce_repo = ce_repo || 'https://gitlab.com/gitlab-org/gitlab-ce.git' + end + + def check + ensure_ee_repo + delete_patches + + generate_patch(ce_branch, ce_patch_full_path) + + Dir.chdir(check_dir) do + step("In the #{check_dir} directory") + + step("Pulling latest master", %w[git pull --ff-only origin master]) + + status = catch(:halt_check) do + ce_branch_compat_check! + + delete_ee_branch_locally + + ee_branch_presence_check! + + ee_branch_compat_check! + end + + delete_ee_branch_locally + delete_patches + + if status.nil? + true + else + false + end + end + end + + private + + def ensure_ee_repo + if Dir.exist?(check_dir) + step("#{check_dir} already exists") + else + cmd = %W[git clone --branch master --single-branch --depth 1 #{EE_REPO} #{check_dir}] + step("Cloning #{EE_REPO} into #{check_dir}", cmd) + end + end + + def ce_branch_compat_check! + cmd = %W[git apply --check #{ce_patch_full_path}] + status = step("Checking if #{ce_patch_name} applies cleanly to EE/master", cmd) + + if status.zero? + puts ce_applies_cleanly_msg(ce_branch) + throw(:halt_check) + end + end + + def ee_branch_presence_check! + status = step("Fetching origin/#{ee_branch}", %W[git fetch origin #{ee_branch}]) + + unless status.zero? + puts + puts ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + + throw(:halt_check, :ko) + end + end + + def ee_branch_compat_check! + step("Checking out origin/#{ee_branch}", %W[git checkout -b #{ee_branch} FETCH_HEAD]) + + generate_patch(ee_branch, ee_patch_full_path) + cmd = %W[git apply --check #{ee_patch_full_path}] + status = step("Checking if #{ee_patch_name} applies cleanly to EE/master", cmd) + + unless status.zero? + puts + puts ee_branch_doesnt_apply_cleanly_msg + + throw(:halt_check, :ko) + end + + puts + puts ee_applies_cleanly_msg + end + + def generate_patch(branch, filepath) + FileUtils.rm(filepath, force: true) + + depth = 0 + loop do + depth += 10 + step("Fetching origin/master", %W[git fetch origin master --depth=#{depth}]) + status = step("Finding merge base with master", %W[git merge-base FETCH_HEAD #{branch}]) + + break if status.zero? || depth > 500 + end + + raise "#{branch} is too far behind master, please rebase it!" if depth > 500 + + step("Generating the patch against master") + output, status = Gitlab::Popen.popen(%w[git format-patch FETCH_HEAD --stdout]) + throw(:halt_check, :ko) unless status.zero? + + File.write(filepath, output) + throw(:halt_check, :ko) unless File.exist?(filepath) + end + + def delete_ee_branch_locally + command(%w[git checkout master]) + step("Deleting the local #{ee_branch} branch", %W[git branch -D #{ee_branch}]) + end + + def delete_patches + step("Deleting #{ce_patch_full_path}") + FileUtils.rm(ce_patch_full_path, force: true) + + step("Deleting #{ee_patch_full_path}") + FileUtils.rm(ee_patch_full_path, force: true) + end + + def ce_patch_name + @ce_patch_name ||= "#{ce_branch}.patch" + end + + def ce_patch_full_path + @ce_patch_full_path ||= File.expand_path(ce_patch_name, check_dir) + end + + def ee_branch + @ee_branch ||= "#{ce_branch}-ee" + end + + def ee_patch_name + @ee_patch_name ||= "#{ee_branch}.patch" + end + + def ee_patch_full_path + @ee_patch_full_path ||= File.expand_path(ee_patch_name, check_dir) + end + + def step(desc, cmd = nil) + puts "\n=> #{desc}\n" + + if cmd + puts "\n$ #{cmd.join(' ')}" + command(cmd) + end + end + + def command(cmd) + output, status = Gitlab::Popen.popen(cmd) + puts output + + status + end + + def ce_applies_cleanly_msg(ce_branch) + <<-MSG.strip_heredoc + ================================================================= + 🎉 Congratulations!! 🎉 + + The #{ce_branch} branch applies cleanly to EE/master! + + Much ❤️!! + =================================================================\n + MSG + end + + def ce_branch_doesnt_apply_cleanly_and_no_ee_branch_msg + <<-MSG.strip_heredoc + ================================================================= + 💥 Oh no! 💥 + + The #{ce_branch} branch does not apply cleanly to the current + EE/master, and no #{ee_branch} branch was found in the EE repository. + + Please create a #{ee_branch} branch that includes changes from + #{ce_branch} but also specific changes than can be applied cleanly + to EE/master. + + There are different ways to create such branch: + + 1. Create a new branch based on the CE branch and rebase it on top of EE/master + + # In the EE repo + $ git fetch #{ce_repo} #{ce_branch} + $ git checkout -b #{ee_branch} FETCH_HEAD + + # You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit + # before rebasing to limit the conflicts-resolving steps during the rebase + $ git fetch origin + $ git rebase origin/master + + At this point you will likely have conflicts. + Solve them, and continue/finish the rebase. + + You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE". + + 2. Create a new branch from master and cherry-pick your CE commits + + # In the EE repo + $ git fetch origin + $ git checkout -b #{ee_branch} FETCH_HEAD + $ git fetch #{ce_repo} #{ce_branch} + $ git cherry-pick SHA # Repeat for all the commits you want to pick + + You can squash the #{ce_branch} commits into a single "Port of #{ce_branch} to EE" commit. + + Don't forget to push your branch to #{EE_REPO}: + + # In the EE repo + $ git push origin #{ee_branch} + + You can then retry this failed build, and hopefully it should pass. + + Stay 💪 ! + =================================================================\n + MSG + end + + def ee_branch_doesnt_apply_cleanly_msg + <<-MSG.strip_heredoc + ================================================================= + 💥 Oh no! 💥 + + The #{ce_branch} does not apply cleanly to the current + EE/master, and even though a #{ee_branch} branch exists in the EE + repository, it does not apply cleanly either to EE/master! + + Please update the #{ee_branch}, push it again to #{EE_REPO}, and + retry this build. + + Stay 💪 ! + =================================================================\n + MSG + end + + def ee_applies_cleanly_msg + <<-MSG.strip_heredoc + ================================================================= + 🎉 Congratulations!! 🎉 + + The #{ee_branch} branch applies cleanly to EE/master! + + Much ❤️!! + =================================================================\n + MSG + end + end +end diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 06dae31cc27..447c7a6a6b9 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -46,7 +46,9 @@ module Gitlab noteable_type: sent_notification.noteable_type, noteable_id: sent_notification.noteable_id, commit_id: sent_notification.commit_id, - line_code: sent_notification.line_code + line_code: sent_notification.line_code, + position: sent_notification.position, + type: sent_notification.note_type ).execute end end diff --git a/lib/gitlab/production_logger.rb b/lib/gitlab/environment_logger.rb index 89ce7144b1b..407cc572656 100644 --- a/lib/gitlab/production_logger.rb +++ b/lib/gitlab/environment_logger.rb @@ -1,7 +1,7 @@ module Gitlab - class ProductionLogger < Gitlab::Logger + class EnvironmentLogger < Gitlab::Logger def self.file_name_noext - 'production' + Rails.env end end end diff --git a/lib/gitlab/exclusive_lease.rb b/lib/gitlab/exclusive_lease.rb index ffe49364379..7e8f35e9298 100644 --- a/lib/gitlab/exclusive_lease.rb +++ b/lib/gitlab/exclusive_lease.rb @@ -27,7 +27,7 @@ module Gitlab # on begin/ensure blocks to cancel a lease, because the 'ensure' does # not always run. Think of 'kill -9' from the Unicorn master for # instance. - # + # # If you find that leases are getting in your way, ask yourself: would # it be enough to lower the lease timeout? Another thing that might be # appropriate is to only use a lease for bulk/automated operations, and @@ -48,6 +48,13 @@ module Gitlab end end + # Returns true if the key for this lease is set. + def exists? + Gitlab::Redis.with do |redis| + redis.exists(redis_key) + end + end + # No #cancel method. See comments above! private diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 65ee85ca5a9..222bcdcbf9c 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -75,7 +75,7 @@ module Gitlab def create_label(name) params = { title: name, color: nice_label_color(name) } - ::Labels::FindOrCreateService.new(project.owner, project, params).execute + ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end def user_info(person_id) @@ -133,7 +133,7 @@ module Gitlab updated_at: DateTime.parse(bug['dtLastUpdated']) ) - issue_labels = ::LabelsFinder.new(project.owner, project_id: project.id, title: labels).execute + issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) issue.update_attribute(:label_ids, issue_labels.pluck(:id)) import_issue_comments(issue, comments) diff --git a/lib/gitlab/github_import/base_formatter.rb b/lib/gitlab/github_import/base_formatter.rb index 8cacf4f4925..6dbae64a9fe 100644 --- a/lib/gitlab/github_import/base_formatter.rb +++ b/lib/gitlab/github_import/base_formatter.rb @@ -10,7 +10,9 @@ module Gitlab end def create! - self.klass.create!(self.attributes) + project.public_send(project_association).find_or_create_by!(find_condition) do |record| + record.attributes = attributes + end end private diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 7f424b74efb..85df6547a67 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -105,18 +105,20 @@ module Gitlab data = api.send(method, *args) return data unless data.is_a?(Array) + last_response = api.last_response + if block_given? yield data - each_response_page(&block) + # api.last_response could change while we're yielding (e.g. fetching labels for each PR) + # so we cache our own last response + each_response_page(last_response, &block) else - each_response_page { |page| data.concat(page) } + each_response_page(last_response) { |page| data.concat(page) } data end end - def each_response_page - last_response = api.last_response - + def each_response_page(last_response) while last_response.rels[:next] sleep rate_limit_sleep_time if rate_limit_exceed? last_response = last_response.rels[:next].get diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb index 4b70f33a851..ecc28799737 100644 --- a/lib/gitlab/github_import/importer.rb +++ b/lib/gitlab/github_import/importer.rb @@ -24,7 +24,8 @@ module Gitlab import_milestones import_issues import_pull_requests - import_comments + import_comments(:issues) + import_comments(:pull_requests) import_wiki import_releases handle_errors @@ -48,7 +49,7 @@ module Gitlab end def import_labels - client.labels(repo, per_page: 100) do |labels| + fetch_resources(:labels, repo, per_page: 100) do |labels| labels.each do |raw| begin label = LabelFormatter.new(project, raw).create! @@ -61,7 +62,7 @@ module Gitlab end def import_milestones - client.milestones(repo, state: :all, per_page: 100) do |milestones| + fetch_resources(:milestones, repo, state: :all, per_page: 100) do |milestones| milestones.each do |raw| begin MilestoneFormatter.new(project, raw).create! @@ -73,7 +74,7 @@ module Gitlab end def import_issues - client.issues(repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| + fetch_resources(:issues, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |issues| issues.each do |raw| gh_issue = IssueFormatter.new(project, raw) @@ -90,7 +91,7 @@ module Gitlab end def import_pull_requests - client.pull_requests(repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| + fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| pull_requests.each do |raw| pull_request = PullRequestFormatter.new(project, raw) next unless pull_request.valid? @@ -132,8 +133,15 @@ module Gitlab end def apply_labels(issuable, raw_issuable) - if raw_issuable.labels.count > 0 - label_ids = raw_issuable.labels + # GH returns labels for issues but not for pull requests! + labels = if issuable.is_a?(MergeRequest) + client.labels_for_issue(repo, raw_issuable.number) + else + raw_issuable.labels + end + + if labels.count > 0 + label_ids = labels .map { |attrs| @labels[attrs.name] } .compact @@ -141,23 +149,35 @@ module Gitlab end end - def import_comments - client.issues_comments(repo, per_page: 100) do |comments| - create_comments(comments, :issue) - end + def import_comments(issuable_type) + resource_type = "#{issuable_type}_comments".to_sym + + # Two notes here: + # 1. We don't have a distinctive attribute for comments (unlike issues iid), so we fetch the last inserted note, + # compare it against every comment in the current imported page until we find match, and that's where start importing + # 2. GH returns comments for _both_ issues and PRs through issues_comments API, while pull_requests_comments returns + # only comments on diffs, so select last note not based on noteable_type but on line_code + line_code_is = issuable_type == :pull_requests ? 'NOT NULL' : 'NULL' + last_note = project.notes.where("line_code IS #{line_code_is}").last + + fetch_resources(resource_type, repo, per_page: 100) do |comments| + if last_note + discard_inserted_comments(comments, last_note) + last_note = nil + end - client.pull_requests_comments(repo, per_page: 100) do |comments| - create_comments(comments, :pull_request) + create_comments(comments) end end - def create_comments(comments, issuable_type) + def create_comments(comments) ActiveRecord::Base.no_touching do comments.each do |raw| begin - comment = CommentFormatter.new(project, raw) - issuable_class = issuable_type == :issue ? Issue : MergeRequest - iid = raw.send("#{issuable_type}_url").split('/').last # GH doesn't return parent ID directly + comment = CommentFormatter.new(project, raw) + # GH does not return info about comment's parent, so we guess it by checking its URL! + *_, parent, iid = URI(raw.html_url).path.split('/') + issuable_class = parent == 'issues' ? Issue : MergeRequest issuable = issuable_class.find_by_iid(iid) next unless issuable @@ -169,6 +189,24 @@ module Gitlab end end + def discard_inserted_comments(comments, last_note) + last_note_attrs = nil + + cut_off_index = comments.find_index do |raw| + comment = CommentFormatter.new(project, raw) + comment_attrs = comment.attributes + last_note_attrs ||= last_note.slice(*comment_attrs.keys) + + comment_attrs.with_indifferent_access == last_note_attrs + end + + # No matching resource in the collection, which means we got halted right on the end of the last page, so all good + return unless cut_off_index + + # Otherwise, remove the resources we've already inserted + comments.shift(cut_off_index + 1) + end + def import_wiki unless project.wiki.repository_exists? wiki = WikiFormatter.new(project) @@ -184,7 +222,7 @@ module Gitlab end def import_releases - client.releases(repo, per_page: 100) do |releases| + fetch_resources(:releases, repo, per_page: 100) do |releases| releases.each do |raw| begin gh_release = ReleaseFormatter.new(project, raw) @@ -195,6 +233,47 @@ module Gitlab end end end + + def fetch_resources(resource_type, *opts) + return if imported?(resource_type) + + opts.last.merge!(page: current_page(resource_type)) + + client.public_send(resource_type, *opts) do |resources| + yield resources + increment_page(resource_type) + end + + imported!(resource_type) + end + + def imported?(resource_type) + Rails.cache.read("#{cache_key_prefix}:#{resource_type}:imported") + end + + def imported!(resource_type) + Rails.cache.write("#{cache_key_prefix}:#{resource_type}:imported", true, ex: 1.day) + end + + def increment_page(resource_type) + key = "#{cache_key_prefix}:#{resource_type}:current-page" + + # Rails.cache.increment calls INCRBY directly on the value stored under the key, which is + # a serialized ActiveSupport::Cache::Entry, so it will return an error by Redis, hence this ugly work-around + page = Rails.cache.read(key) + page += 1 + Rails.cache.write(key, page) + + page + end + + def current_page(resource_type) + Rails.cache.fetch("#{cache_key_prefix}:#{resource_type}:current-page", ex: 1.day) { 1 } + end + + def cache_key_prefix + @cache_key_prefix ||= "github-import:#{project.id}" + end end end end diff --git a/lib/gitlab/github_import/issue_formatter.rb b/lib/gitlab/github_import/issue_formatter.rb index 77621de9f4c..8c32ac59fc5 100644 --- a/lib/gitlab/github_import/issue_formatter.rb +++ b/lib/gitlab/github_import/issue_formatter.rb @@ -20,8 +20,12 @@ module Gitlab raw_data.comments > 0 end - def klass - Issue + def project_association + :issues + end + + def find_condition + { iid: number } end def number diff --git a/lib/gitlab/github_import/label_formatter.rb b/lib/gitlab/github_import/label_formatter.rb index 942dfb3312b..211ccdc51bb 100644 --- a/lib/gitlab/github_import/label_formatter.rb +++ b/lib/gitlab/github_import/label_formatter.rb @@ -9,14 +9,14 @@ module Gitlab } end - def klass - Label + def project_association + :labels end def create! params = attributes.except(:project) - service = ::Labels::FindOrCreateService.new(project.owner, project, params) - label = service.execute + service = ::Labels::FindOrCreateService.new(nil, project, params) + label = service.execute(skip_authorization: true) raise ActiveRecord::RecordInvalid.new(label) unless label.persisted? diff --git a/lib/gitlab/github_import/milestone_formatter.rb b/lib/gitlab/github_import/milestone_formatter.rb index b2fa524cf5b..401dd962521 100644 --- a/lib/gitlab/github_import/milestone_formatter.rb +++ b/lib/gitlab/github_import/milestone_formatter.rb @@ -14,8 +14,12 @@ module Gitlab } end - def klass - Milestone + def project_association + :milestones + end + + def find_condition + { iid: raw_data.number } end private diff --git a/lib/gitlab/github_import/pull_request_formatter.rb b/lib/gitlab/github_import/pull_request_formatter.rb index 1408683100f..b9a227fb11a 100644 --- a/lib/gitlab/github_import/pull_request_formatter.rb +++ b/lib/gitlab/github_import/pull_request_formatter.rb @@ -24,8 +24,12 @@ module Gitlab } end - def klass - MergeRequest + def project_association + :merge_requests + end + + def find_condition + { iid: number } end def number diff --git a/lib/gitlab/github_import/release_formatter.rb b/lib/gitlab/github_import/release_formatter.rb index 73d643b00ad..1ad702a6058 100644 --- a/lib/gitlab/github_import/release_formatter.rb +++ b/lib/gitlab/github_import/release_formatter.rb @@ -11,8 +11,12 @@ module Gitlab } end - def klass - Release + def project_association + :releases + end + + def find_condition + { tag: raw_data.tag_name } end def valid? diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index 6a68e786b4f..1f4edc36928 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -101,7 +101,7 @@ module Gitlab state: raw_issue['state'] == 'closed' ? 'closed' : 'opened' ) - issue_labels = ::LabelsFinder.new(project.owner, project_id: project.id, title: labels).execute + issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) issue.update_attribute(:label_ids, issue_labels.pluck(:id)) import_issue_comments(issue, comments) @@ -235,7 +235,7 @@ module Gitlab def create_label(name) params = { name: name, color: nice_label_color(name) } - ::Labels::FindOrCreateService.new(project.owner, project, params).execute + ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end def format_content(raw_content) diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index f755a404693..34169319b26 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -3,10 +3,25 @@ module Gitlab class AttributeCleaner ALLOWED_REFERENCES = RelationFactory::PROJECT_REFERENCES + RelationFactory::USER_REFERENCES + ['group_id'] - def self.clean!(relation_hash:) - relation_hash.reject! do |key, _value| - key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) - end + def self.clean(*args) + new(*args).clean + end + + def initialize(relation_hash:, relation_class:) + @relation_hash = relation_hash + @relation_class = relation_class + end + + def clean + @relation_hash.reject do |key, _value| + prohibited_key?(key) || !@relation_class.attribute_method?(key) + end.except('id') + end + + private + + def prohibited_key?(key) + key.end_with?('_id') && !ALLOWED_REFERENCES.include?(key) end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 113895ba22c..ffd17118c91 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -43,6 +43,14 @@ module Gitlab raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result + remove_symlinks! + end + + def remove_symlinks! + Dir["#{@shared.export_path}/**/*"].each do |path| + FileUtils.rm(path) if File.lstat(path).symlink? + end + true end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 36c4cf6efa0..b790733f4a7 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -55,7 +55,12 @@ module Gitlab end def member_hash(member) - member.except('id').merge(source_id: @project.id, importing: true) + parsed_hash(member).merge('source_id' => @project.id, 'importing' => true) + end + + def parsed_hash(member) + Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, + relation_class: ProjectMember) end def find_project_user_query(member) diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb index 7cdba880a93..c551321c18d 100644 --- a/lib/gitlab/import_export/project_tree_restorer.rb +++ b/lib/gitlab/import_export/project_tree_restorer.rb @@ -9,8 +9,14 @@ module Gitlab end def restore - json = IO.read(@path) - @tree_hash = ActiveSupport::JSON.decode(json) + begin + json = IO.read(@path) + @tree_hash = ActiveSupport::JSON.decode(json) + rescue => e + Rails.logger.error("Import/Export error: #{e.message}") + raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + end + @project_members = @tree_hash.delete('project_members') ActiveRecord::Base.no_touching do diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb index dc630e76411..a0e80fccad9 100644 --- a/lib/gitlab/import_export/relation_factory.rb +++ b/lib/gitlab/import_export/relation_factory.rb @@ -14,7 +14,7 @@ module Gitlab priorities: :label_priorities, label: :project_label }.freeze - USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id].freeze + USER_REFERENCES = %w[author_id assignee_id updated_by_id user_id created_by_id].freeze PROJECT_REFERENCES = %w[project_id source_project_id gl_project_id target_project_id].freeze @@ -30,7 +30,7 @@ module Gitlab def initialize(relation_sym:, relation_hash:, members_mapper:, user:, project_id:) @relation_name = OVERRIDES[relation_sym] || relation_sym - @relation_hash = relation_hash.except('id', 'noteable_id').merge('project_id' => project_id) + @relation_hash = relation_hash.except('noteable_id').merge('project_id' => project_id) @members_mapper = members_mapper @user = user @imported_object_retries = 0 @@ -172,11 +172,8 @@ module Gitlab end def parsed_relation_hash - @parsed_relation_hash ||= begin - Gitlab::ImportExport::AttributeCleaner.clean!(relation_hash: @relation_hash) - - @relation_hash.reject { |k, _v| !relation_class.attribute_method?(k) } - end + @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash, + relation_class: relation_class) end def set_st_diffs diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index fc08082fc86..bd3c3ee3b2f 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -24,12 +24,19 @@ module Gitlab end def verify_version!(version) - if Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) + if different_version?(version) raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}") else true end end + + def different_version?(version) + Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) + rescue => e + Rails.logger.error("Import/Export error: #{e.message}") + raise Gitlab::ImportExport::Error.new('Incorrect VERSION format') + end end end end diff --git a/lib/gitlab/issues_labels.rb b/lib/gitlab/issues_labels.rb index 01a2c19ab23..b8ca7f2f55f 100644 --- a/lib/gitlab/issues_labels.rb +++ b/lib/gitlab/issues_labels.rb @@ -19,7 +19,7 @@ module Gitlab ] labels.each do |params| - ::Labels::FindOrCreateService.new(project.owner, project).execute(params) + ::Labels::FindOrCreateService.new(nil, project, params).execute(skip_authorization: true) end end end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index 12999a90a29..a5220d92312 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -33,7 +33,12 @@ module Gitlab config[:mailbox] = 'inbox' if config[:mailbox].nil? if config[:enabled] && config[:address] - config[:redis_url] = Gitlab::Redis.new(rails_env).url + gitlab_redis = Gitlab::Redis.new(rails_env) + config[:redis_url] = gitlab_redis.url + + if gitlab_redis.sentinels? + config[:sentinels] = gitlab_redis.sentinels + end end config diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb new file mode 100644 index 00000000000..879d46446b3 --- /dev/null +++ b/lib/gitlab/optimistic_locking.rb @@ -0,0 +1,19 @@ +module Gitlab + module OptimisticLocking + extend self + + def retry_lock(subject, retries = 100, &block) + loop do + begin + ActiveRecord::Base.transaction do + return block.call(subject) + end + rescue ActiveRecord::StaleObjectError + retries -= 1 + raise unless retries >= 0 + subject.reload + end + end + end + end +end diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index 5b9cfaeb2f8..24733435a5a 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -73,11 +73,7 @@ module Gitlab end def commits - if project.empty_repo? || query.blank? - [] - else - project.repository.find_commits_by_message(query).compact - end + project.repository.find_commits_by_message(query) end def project_ids_relation diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index c649da8c426..9226da2d6b1 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -63,6 +63,14 @@ module Gitlab raw_config_hash[:url] end + def sentinels + raw_config_hash[:sentinels] + end + + def sentinels? + sentinels && !sentinels.empty? + end + private def redis_store_options diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index e59ead5d76c..4c395b4266e 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -13,5 +13,13 @@ module Gitlab def force_utf8(str) str.force_encoding(Encoding::UTF_8) end + + def to_boolean(value) + return value if [true, false].include?(value) + return true if value =~ /^(true|t|yes|y|1|on)$/i + return false if value =~ /^(false|f|no|n|0|off)$/i + + nil + end end end diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index a95a3455a4a..78ae187817a 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -29,5 +29,5 @@ namespace :cache do task all: [:db, :redis] end - task clear: 'cache:clear:all' + task clear: 'cache:clear:redis' end diff --git a/lib/tasks/ce_to_ee_merge_check.rake b/lib/tasks/ce_to_ee_merge_check.rake deleted file mode 100644 index 424e7883060..00000000000 --- a/lib/tasks/ce_to_ee_merge_check.rake +++ /dev/null @@ -1,4 +0,0 @@ -desc 'Checks if the branch would apply cleanly to EE' -task ce_to_ee_merge_check: :environment do - Rake::Task['gitlab:dev:ce_to_ee_merge_check'].invoke -end diff --git a/lib/tasks/ee_compat_check.rake b/lib/tasks/ee_compat_check.rake new file mode 100644 index 00000000000..f494fa5c5c2 --- /dev/null +++ b/lib/tasks/ee_compat_check.rake @@ -0,0 +1,4 @@ +desc 'Checks if the branch would apply cleanly to EE' +task ee_compat_check: :environment do + Rake::Task['gitlab:dev:ee_compat_check'].invoke +end diff --git a/lib/tasks/eslint.rake b/lib/tasks/eslint.rake new file mode 100644 index 00000000000..d43cbad1909 --- /dev/null +++ b/lib/tasks/eslint.rake @@ -0,0 +1,7 @@ +unless Rails.env.production? + desc "GitLab | Run ESLint" + task :eslint do + system("npm", "run", "eslint") + end +end + diff --git a/lib/tasks/gitlab/backup.rake b/lib/tasks/gitlab/backup.rake index b43ee5b3383..a9f1255e8cf 100644 --- a/lib/tasks/gitlab/backup.rake +++ b/lib/tasks/gitlab/backup.rake @@ -51,6 +51,7 @@ namespace :gitlab do $progress.puts 'done'.color(:green) Rake::Task['gitlab:backup:db:restore'].invoke end + Rake::Task['gitlab:backup:repo:restore'].invoke unless backup.skipped?('repositories') Rake::Task['gitlab:backup:uploads:restore'].invoke unless backup.skipped?('uploads') Rake::Task['gitlab:backup:builds:restore'].invoke unless backup.skipped?('builds') @@ -58,6 +59,7 @@ namespace :gitlab do Rake::Task['gitlab:backup:lfs:restore'].invoke unless backup.skipped?('lfs') Rake::Task['gitlab:backup:registry:restore'].invoke unless backup.skipped?('registry') Rake::Task['gitlab:shell:setup'].invoke + Rake::Task['cache:clear'].invoke backup.cleanup end diff --git a/lib/tasks/gitlab/dev.rake b/lib/tasks/gitlab/dev.rake index 47bdb2d32d2..5ee99dfc810 100644 --- a/lib/tasks/gitlab/dev.rake +++ b/lib/tasks/gitlab/dev.rake @@ -1,106 +1,21 @@ namespace :gitlab do namespace :dev do desc 'Checks if the branch would apply cleanly to EE' - task ce_to_ee_merge_check: :environment do + task ee_compat_check: :environment do return if defined?(Gitlab::License) return unless ENV['CI'] - ce_repo = ENV['CI_BUILD_REPO'] - ce_branch = ENV['CI_BUILD_REF_NAME'] - - ee_repo = 'https://gitlab.com/gitlab-org/gitlab-ee.git' - ee_branch = "#{ce_branch}-ee" - ee_dir = 'gitlab-ee-merge-check' - - puts "\n=> Cloning #{ee_repo} into #{ee_dir}\n" - `git clone #{ee_repo} #{ee_dir} --depth 1` - Dir.chdir(ee_dir) do - puts "\n => Fetching #{ce_repo}/#{ce_branch}\n" - `git fetch #{ce_repo} #{ce_branch} --depth 1` - - # Try to merge the current tested branch to EE/master... - puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" - `git merge FETCH_HEAD` - - exit 0 if $?.success? - - # Check if the <branch>-ee branch exists... - puts "\n => Check if #{ee_repo}/#{ee_branch} exists\n" - `git rev-parse --verify #{ee_branch}` - - # The <branch>-ee doesn't exist - unless $?.success? - puts - puts <<-MSG.strip_heredoc - ================================================================= - The #{ce_branch} branch cannot be merged without conflicts to the - current EE/master, and no #{ee_branch} branch was detected in - the EE repository. - - Please create a #{ee_branch} branch that includes changes from - #{ce_branch} but also specific changes than can be applied cleanly - to EE/master. - - You can create this branch as follows: - - 1. In the EE repo: - $ git fetch origin - $ git fetch #{ce_repo} #{ce_branch} - $ git checkout -b #{ee_branch} FETCH_HEAD - $ git rebase origin/master - 2. At this point you will likely have conflicts, solve them, and - continue/finish the rebase. Note: You can squash the CE commits - before rebasing. - 3. You can squash all the original #{ce_branch} commits into a - single "Port of #{ce_branch} to EE". - 4. Push your branch to #{ee_repo}: - $ git push origin #{ee_branch} - =================================================================\n - MSG - - exit 1 - end - - # Try to merge the <branch>-ee branch to EE/master... - puts "\n => Merging #{ee_repo}/#{ee_branch} into #{ee_repo}/master\n" - `git merge #{ee_branch} master` - - # The <branch>-ee cannot be merged cleanly to EE/master... - unless $?.success? - puts - puts <<-MSG.strip_heredoc - ================================================================= - The #{ce_branch} branch cannot be merged without conflicts to - EE/master, and even though the #{ee_branch} branch exists in the EE - repository, it cannot be merged without conflicts to EE/master. - - Please update the #{ee_branch}, push it again to #{ee_repo}, and - retry this job. - =================================================================\n - MSG - - exit 2 - end - - puts "\n => Merging #{ce_repo}/#{ce_branch} into #{ee_repo}/master\n" - `git merge FETCH_HEAD` - exit 0 if $?.success? - - # The <branch>-ee can be merged cleanly to EE/master, but <branch> still - # cannot be merged cleanly to EE/master... - puts - puts <<-MSG.strip_heredoc - ================================================================= - The #{ce_branch} branch cannot be merged without conflicts to EE, and - even though the #{ee_branch} branch exists in the EE repository and - applies cleanly to EE/master, it doesn't prevent conflicts when - merging #{ce_branch} into EE. - - We may be in a complex situation here. - =================================================================\n - MSG - - exit 3 + success = + Gitlab::EeCompatCheck.new( + branch: ENV['CI_BUILD_REF_NAME'], + check_dir: File.expand_path('ee-compat-check', __dir__), + ce_repo: ENV['CI_BUILD_REPO'] + ).check + + if success + exit 0 + else + exit 1 end end end diff --git a/lib/tasks/gitlab/generate_docs.rake b/lib/tasks/gitlab/generate_docs.rake deleted file mode 100644 index f6448c38e10..00000000000 --- a/lib/tasks/gitlab/generate_docs.rake +++ /dev/null @@ -1,7 +0,0 @@ -namespace :gitlab do - desc "GitLab | Generate sdocs for project" - task generate_docs: :environment do - system(*%W(bundle exec sdoc -o doc/code app lib)) - end -end - diff --git a/lib/tasks/gitlab/shell.rake b/lib/tasks/gitlab/shell.rake index 210899882b4..58761a129d4 100644 --- a/lib/tasks/gitlab/shell.rake +++ b/lib/tasks/gitlab/shell.rake @@ -63,11 +63,11 @@ namespace :gitlab do # Launch installation process system(*%W(bin/install) + repository_storage_paths_args) - - # (Re)create hooks - system(*%W(bin/create-hooks) + repository_storage_paths_args) end + # (Re)create hooks + Rake::Task['gitlab:shell:create_hooks'].invoke + # Required for debian packaging with PKGR: Setup .ssh/environment with # the current PATH, so that the correct ruby version gets loaded # Requires to set "PermitUserEnvironment yes" in sshd config (should not @@ -102,6 +102,15 @@ namespace :gitlab do end end end + + desc 'Create or repair repository hooks symlink' + task create_hooks: :environment do + warn_user_is_not_gitlab + + puts 'Creating/Repairing hooks symlinks for all repositories' + system(*%W(#{Gitlab.config.gitlab_shell.path}/bin/create-hooks) + repository_storage_paths_args) + puts 'done'.color(:green) + end end def setup diff --git a/lib/tasks/lint.rake b/lib/tasks/lint.rake new file mode 100644 index 00000000000..32b668df3bf --- /dev/null +++ b/lib/tasks/lint.rake @@ -0,0 +1,9 @@ +unless Rails.env.production? + namespace :lint do + desc "GitLab | lint | Lint JavaScript files using ESLint" + task :javascript do + Rake::Task['eslint'].invoke + end + end +end + diff --git a/lib/tasks/teaspoon.rake b/lib/tasks/teaspoon.rake new file mode 100644 index 00000000000..08caedd7ff3 --- /dev/null +++ b/lib/tasks/teaspoon.rake @@ -0,0 +1,25 @@ +unless Rails.env.production? + Rake::Task['teaspoon'].clear if Rake::Task.task_defined?('teaspoon') + + namespace :teaspoon do + desc 'GitLab | Teaspoon | Generate fixtures for JavaScript tests' + RSpec::Core::RakeTask.new(:fixtures) do |t| + ENV['NO_KNAPSACK'] = 'true' + t.pattern = 'spec/javascripts/fixtures/*.rb' + t.rspec_opts = '--format documentation' + end + + desc 'GitLab | Teaspoon | Run JavaScript tests' + task :tests do + require "teaspoon/console" + options = {} + abort('rake teaspoon:tests failed') if Teaspoon::Console.new(options).failures? + end + end + + desc 'GitLab | Teaspoon | Shortcut for teaspoon:fixtures and teaspoon:tests' + task :teaspoon do + Rake::Task['teaspoon:fixtures'].invoke + Rake::Task['teaspoon:tests'].invoke + end +end |