diff options
author | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2017-06-05 13:51:59 +0300 |
---|---|---|
committer | Grzegorz Bizon <grzesiek.bizon@gmail.com> | 2017-06-05 13:51:59 +0300 |
commit | 79380fe5fb5943179960f4879e174eb977b918d3 (patch) | |
tree | 94953113c8bf5174cf52fb78d90e28a8c14c20c7 /app/services | |
parent | 3801a0df80306a76dc340ca74427a124a1514dbb (diff) | |
parent | c34107608ecc5c36e80a748eb4c9b88d2b1157cf (diff) |
Merge branch 'master' into feature/gb/persist-pipeline-stages
* master: (524 commits)
Improve user experience around slash commands in instant comments
Fix LFS timeouts when trying to save large files
retryable? is now available for CommitStatus
Resolve "Documentation of `.gitlab-ci.yml` states incorrect example for variables"
Fix test failures
Add slugify project path to CI enviroment variables
Fixed typo: craeted -> created
32118 Make New environment empty state btn lowercase
Expose import_status in Projects API
32832 Remove overflow from comment form for confidential issues and vertically aligns confidential issue icon
Fix test failures
Allow manual bypass of auto_sign_in_with_provider
Fix keys seed
Allow users to be hard-deleted from the API
fixup some classnames and media queries
Enable the Style/PreferredHashMethods cop
Lint our factories creation in addition to their build
Don’t schedule workers from inside transactions
Allow scheduling from after_commit hooks
Forbid Sidekiq scheduling in transactions
...
Conflicts:
app/serializers/pipeline_entity.rb
db/schema.rb
spec/factories/ci/stages.rb
spec/lib/gitlab/import_export/safe_model_attributes.yml
spec/services/ci/create_pipeline_service_spec.rb
spec/spec_helper.rb
Diffstat (limited to 'app/services')
21 files changed, 303 insertions, 140 deletions
diff --git a/app/services/ci/create_pipeline_service.rb b/app/services/ci/create_pipeline_service.rb index 5ff388c6614..bffec216819 100644 --- a/app/services/ci/create_pipeline_service.rb +++ b/app/services/ci/create_pipeline_service.rb @@ -2,8 +2,9 @@ module Ci class CreatePipelineService < BaseService attr_reader :pipeline - def execute(ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil) + def execute(source, ignore_skip_ci: false, save_on_errors: true, trigger_request: nil, schedule: nil) @pipeline = Ci::Pipeline.new( + source: source, project: project, ref: ref, sha: sha, @@ -61,6 +62,13 @@ module Ci private + def update_merge_requests_head_pipeline + return unless pipeline.latest? + + MergeRequest.where(source_project: @pipeline.project, source_branch: @pipeline.ref). + update_all(head_pipeline_id: @pipeline.id) + end + def skip_ci? return false unless pipeline.git_commit_message pipeline.git_commit_message =~ /\[(ci[ _-]skip|skip[ _-]ci)\]/i @@ -118,11 +126,6 @@ module Ci origin_sha && origin_sha != Gitlab::Git::BLANK_SHA end - def update_merge_requests_head_pipeline - MergeRequest.where(source_branch: @pipeline.ref, source_project: @pipeline.project). - update_all(head_pipeline_id: @pipeline.id) - end - def error(message, save: false) pipeline.errors.add(:base, message) pipeline.drop if save diff --git a/app/services/ci/create_trigger_request_service.rb b/app/services/ci/create_trigger_request_service.rb index 8362f01ddb8..beb27a5a597 100644 --- a/app/services/ci/create_trigger_request_service.rb +++ b/app/services/ci/create_trigger_request_service.rb @@ -4,7 +4,7 @@ module Ci trigger_request = trigger.trigger_requests.create(variables: variables) pipeline = Ci::CreatePipelineService.new(project, trigger.owner, ref: ref). - execute(ignore_skip_ci: true, trigger_request: trigger_request) + execute(:trigger, ignore_skip_ci: true, trigger_request: trigger_request) trigger_request if pipeline.persisted? end diff --git a/app/services/create_deployment_service.rb b/app/services/create_deployment_service.rb index 47f9b2c621c..46823418bb0 100644 --- a/app/services/create_deployment_service.rb +++ b/app/services/create_deployment_service.rb @@ -1,71 +1,59 @@ -class CreateDeploymentService < BaseService - def execute(deployable = nil) +class CreateDeploymentService + attr_reader :job + + delegate :expanded_environment_name, + :environment_url, + :project, + to: :job + + def initialize(job) + @job = job + end + + def execute return unless executable? ActiveRecord::Base.transaction do - @deployable = deployable + environment.external_url = environment_url if environment_url + environment.fire_state_event(action) - @environment = environment - @environment.external_url = expanded_url if expanded_url - @environment.fire_state_event(action) + return unless environment.save + return if environment.stopped? - return unless @environment.save - return if @environment.stopped? - - deploy.tap do |deployment| - deployment.update_merge_request_metrics! - end + deploy.tap(&:update_merge_request_metrics!) end end private def executable? - project && name.present? + project && job.environment.present? && environment end def deploy project.deployments.create( - environment: @environment, - ref: params[:ref], - tag: params[:tag], - sha: params[:sha], - user: current_user, - deployable: @deployable, - on_stop: options[:on_stop]) + environment: environment, + ref: job.ref, + tag: job.tag, + sha: job.sha, + user: job.user, + deployable: job, + on_stop: on_stop) end def environment - @environment ||= project.environments.find_or_create_by(name: expanded_name) - end - - def expanded_name - ExpandVariables.expand(name, variables) - end - - def expanded_url - return unless url - - @expanded_url ||= ExpandVariables.expand(url, variables) - end - - def name - params[:environment] - end - - def url - options[:url] + @environment ||= job.persisted_environment end - def options - params[:options] || {} + def environment_options + @environment_options ||= job.options&.dig(:environment) || {} end - def variables - params[:variables] || [] + def on_stop + environment_options[:on_stop] end def action - options[:action] || 'start' + environment_options[:action] || 'start' end end diff --git a/app/services/discussions/update_diff_position_service.rb b/app/services/discussions/update_diff_position_service.rb new file mode 100644 index 00000000000..1ef8d9edbe1 --- /dev/null +++ b/app/services/discussions/update_diff_position_service.rb @@ -0,0 +1,41 @@ +module Discussions + class UpdateDiffPositionService < BaseService + def execute(discussion) + result = tracer.trace(discussion.position) + return unless result + + position = result[:position] + outdated = result[:outdated] + + discussion.notes.each do |note| + if outdated + note.change_position = position + else + note.position = position + note.change_position = nil + end + end + + Note.transaction do + discussion.notes.each do |note| + Gitlab::Timeless.timeless(note, &:save) + end + + if outdated && current_user + SystemNoteService.diff_discussion_outdated(discussion, project, current_user, position) + end + end + end + + private + + def tracer + @tracer ||= Gitlab::Diff::PositionTracer.new( + project: project, + old_diff_refs: params[:old_diff_refs], + new_diff_refs: params[:new_diff_refs], + paths: params[:paths] + ) + end + end +end diff --git a/app/services/git_push_service.rb b/app/services/git_push_service.rb index d22236b961b..f080e6326a1 100644 --- a/app/services/git_push_service.rb +++ b/app/services/git_push_service.rb @@ -106,7 +106,7 @@ class GitPushService < BaseService EventCreateService.new.push(@project, current_user, build_push_data) @project.execute_hooks(build_push_data.dup, :push_hooks) @project.execute_services(build_push_data.dup, :push_hooks) - Ci::CreatePipelineService.new(@project, current_user, build_push_data).execute + Ci::CreatePipelineService.new(@project, current_user, build_push_data).execute(:push) if push_remove_branch? AfterBranchDeleteService diff --git a/app/services/git_tag_push_service.rb b/app/services/git_tag_push_service.rb index 96432837481..7c424fba428 100644 --- a/app/services/git_tag_push_service.rb +++ b/app/services/git_tag_push_service.rb @@ -11,7 +11,7 @@ class GitTagPushService < BaseService SystemHooksService.new.execute_hooks(build_system_push_data.dup, :tag_push_hooks) project.execute_hooks(@push_data.dup, :tag_push_hooks) project.execute_services(@push_data.dup, :tag_push_hooks) - Ci::CreatePipelineService.new(project, current_user, @push_data).execute + Ci::CreatePipelineService.new(project, current_user, @push_data).execute(:push) ProjectCacheWorker.perform_async(project.id, [], [:commit_count, :repository_size]) true diff --git a/app/services/gravatar_service.rb b/app/services/gravatar_service.rb index 433ecc2df32..e77e08aa380 100644 --- a/app/services/gravatar_service.rb +++ b/app/services/gravatar_service.rb @@ -1,15 +1,20 @@ class GravatarService include Gitlab::CurrentSettings - def execute(email, size = nil, scale = 2) - if current_application_settings.gravatar_enabled? && email.present? - size = 40 if size.nil? || size <= 0 + def execute(email, size = nil, scale = 2, username: nil) + return unless current_application_settings.gravatar_enabled? - sprintf gravatar_url, - hash: Digest::MD5.hexdigest(email.strip.downcase), - size: size * scale, - email: email.strip - end + identifier = email.presence || username.presence + return unless identifier + + hash = Digest::MD5.hexdigest(identifier.strip.downcase) + size = 40 unless size && size > 0 + + sprintf gravatar_url, + hash: hash, + size: size * scale, + email: ERB::Util.url_encode(email&.strip || ''), + username: ERB::Util.url_encode(username&.strip || '') end def gitlab_config diff --git a/app/services/issuable_base_service.rb b/app/services/issuable_base_service.rb index e94ab3e64db..e77a3e3eac1 100644 --- a/app/services/issuable_base_service.rb +++ b/app/services/issuable_base_service.rb @@ -148,7 +148,7 @@ class IssuableBaseService < BaseService execute(params[:description], issuable) # Avoid a description already set on an issuable to be overwritten by a nil - params[:description] = description if params.has_key?(:description) + params[:description] = description if params.key?(:description) params.merge!(command_params) end diff --git a/app/services/issues/close_service.rb b/app/services/issues/close_service.rb index f1030912c68..85c616ca576 100644 --- a/app/services/issues/close_service.rb +++ b/app/services/issues/close_service.rb @@ -28,6 +28,7 @@ module Issues notification_service.close_issue(issue, current_user) if notifications todo_service.close_issue(issue, current_user) execute_hooks(issue, 'close') + invalidate_cache_counts(issue.assignees, issue) end issue diff --git a/app/services/issues/reopen_service.rb b/app/services/issues/reopen_service.rb index 40fbe354492..80ea6312768 100644 --- a/app/services/issues/reopen_service.rb +++ b/app/services/issues/reopen_service.rb @@ -8,6 +8,7 @@ module Issues create_note(issue) notification_service.reopen_issue(issue, current_user) execute_hooks(issue, 'reopen') + invalidate_cache_counts(issue.assignees, issue) end issue diff --git a/app/services/merge_requests/close_service.rb b/app/services/merge_requests/close_service.rb index f2053bda83a..2ffc989ed71 100644 --- a/app/services/merge_requests/close_service.rb +++ b/app/services/merge_requests/close_service.rb @@ -13,6 +13,7 @@ module MergeRequests notification_service.close_mr(merge_request, current_user) todo_service.close_merge_request(merge_request, current_user) execute_hooks(merge_request, 'close') + invalidate_cache_counts(merge_request.assignees, merge_request) end merge_request diff --git a/app/services/merge_requests/conflicts/resolve_service.rb b/app/services/merge_requests/conflicts/resolve_service.rb index d74a82effd6..c2c335b8461 100644 --- a/app/services/merge_requests/conflicts/resolve_service.rb +++ b/app/services/merge_requests/conflicts/resolve_service.rb @@ -37,11 +37,13 @@ module MergeRequests private def write_resolved_file_to_index(merge_index, rugged, file, params) - new_file = if params[:sections] - file.resolve_lines(params[:sections]).map(&:text).join("\n") - elsif params[:content] - file.resolve_content(params[:content]) - end + if params[:sections] + new_file = file.resolve_lines(params[:sections]).map(&:text).join("\n") + + new_file << "\n" if file.our_blob.data.ends_with?("\n") + elsif params[:content] + new_file = file.resolve_content(params[:content]) + end our_path = file.our_path diff --git a/app/services/merge_requests/create_service.rb b/app/services/merge_requests/create_service.rb index b0ae2dfe4ce..71d37797bb4 100644 --- a/app/services/merge_requests/create_service.rb +++ b/app/services/merge_requests/create_service.rb @@ -11,7 +11,9 @@ module MergeRequests merge_request = MergeRequest.new merge_request.source_project = source_project + merge_request.source_branch = params[:source_branch] merge_request.merge_params['force_remove_source_branch'] = params.delete(:force_remove_source_branch) + merge_request.head_pipeline = head_pipeline_for(merge_request) create(merge_request) end @@ -22,5 +24,18 @@ module MergeRequests todo_service.new_merge_request(issuable, current_user) issuable.cache_merge_request_closes_issues!(current_user) end + + private + + def head_pipeline_for(merge_request) + return unless merge_request.source_project + + sha = merge_request.source_branch_sha + return unless sha + + pipelines = merge_request.source_project.pipelines.where(ref: merge_request.source_branch, sha: sha) + + pipelines.order(id: :desc).first + end end end diff --git a/app/services/merge_requests/post_merge_service.rb b/app/services/merge_requests/post_merge_service.rb index e8fb1b59752..f0d998731d7 100644 --- a/app/services/merge_requests/post_merge_service.rb +++ b/app/services/merge_requests/post_merge_service.rb @@ -13,6 +13,7 @@ module MergeRequests create_note(merge_request) notification_service.merge_mr(merge_request, current_user) execute_hooks(merge_request, 'merge') + invalidate_cache_counts(merge_request.assignees, merge_request) end private diff --git a/app/services/merge_requests/reopen_service.rb b/app/services/merge_requests/reopen_service.rb index 54b19e6d651..f2fddf7f345 100644 --- a/app/services/merge_requests/reopen_service.rb +++ b/app/services/merge_requests/reopen_service.rb @@ -10,6 +10,7 @@ module MergeRequests execute_hooks(merge_request, 'reopen') merge_request.reload_diff(current_user) merge_request.mark_as_unchecked + invalidate_cache_counts(merge_request.assignees, merge_request) end merge_request diff --git a/app/services/notes/diff_position_update_service.rb b/app/services/notes/diff_position_update_service.rb deleted file mode 100644 index eff7b287269..00000000000 --- a/app/services/notes/diff_position_update_service.rb +++ /dev/null @@ -1,33 +0,0 @@ -module Notes - class DiffPositionUpdateService < BaseService - def execute(note) - results = tracer.trace(note.position) - return unless results - - position = results[:position] - outdated = results[:outdated] - - if outdated - note.change_position = position - - if note.persisted? && current_user - SystemNoteService.diff_discussion_outdated(note.to_discussion, project, current_user, position) - end - else - note.position = position - note.change_position = nil - end - end - - private - - def tracer - @tracer ||= Gitlab::Diff::PositionTracer.new( - project: project, - old_diff_refs: params[:old_diff_refs], - new_diff_refs: params[:new_diff_refs], - paths: params[:paths] - ) - end - end -end diff --git a/app/services/projects/destroy_service.rb b/app/services/projects/destroy_service.rb index 06d8d143231..e2b2660ea71 100644 --- a/app/services/projects/destroy_service.rb +++ b/app/services/projects/destroy_service.rb @@ -7,11 +7,9 @@ module Projects DELETED_FLAG = '+deleted'.freeze def async_execute - project.transaction do - project.update_attribute(:pending_delete, true) - job_id = ProjectDestroyWorker.perform_async(project.id, current_user.id, params) - Rails.logger.info("User #{current_user.id} scheduled destruction of project #{project.path_with_namespace} with job ID #{job_id}") - end + project.update_attribute(:pending_delete, true) + job_id = ProjectDestroyWorker.perform_async(project.id, current_user.id, params) + Rails.logger.info("User #{current_user.id} scheduled destruction of project #{project.path_with_namespace} with job ID #{job_id}") end def execute @@ -62,7 +60,11 @@ module Projects if gitlab_shell.mv_repository(project.repository_storage_path, path, new_path) log_info("Repository \"#{path}\" moved to \"#{new_path}\"") - GitlabShellWorker.perform_in(5.minutes, :remove_repository, project.repository_storage_path, new_path) + + project.run_after_commit do + # self is now project + GitlabShellWorker.perform_in(5.minutes, :remove_repository, self.repository_storage_path, new_path) + end else false end diff --git a/app/services/search_service.rb b/app/services/search_service.rb index 22736c71725..1d4d03a8b7d 100644 --- a/app/services/search_service.rb +++ b/app/services/search_service.rb @@ -12,7 +12,7 @@ class SearchService @project = if params[:project_id].present? the_project = Project.find_by(id: params[:project_id]) - can?(current_user, :download_code, the_project) ? the_project : nil + can?(current_user, :read_project, the_project) ? the_project : nil else nil end diff --git a/app/services/submit_usage_ping_service.rb b/app/services/submit_usage_ping_service.rb new file mode 100644 index 00000000000..17857ca62f2 --- /dev/null +++ b/app/services/submit_usage_ping_service.rb @@ -0,0 +1,41 @@ +class SubmitUsagePingService + URL = 'https://version.gitlab.com/usage_data'.freeze + + include Gitlab::CurrentSettings + + def execute + return false unless current_application_settings.usage_ping_enabled? + + response = HTTParty.post( + URL, + body: Gitlab::UsageData.to_json(force_refresh: true), + headers: { 'Content-type' => 'application/json' } + ) + + store_metrics(response) + + true + rescue HTTParty::Error => e + Rails.logger.info "Unable to contact GitLab, Inc.: #{e}" + + false + end + + private + + def store_metrics(response) + return unless response['conv_index'].present? + + ConversationalDevelopmentIndex::Metric.create!( + response['conv_index'].slice( + 'leader_issues', 'instance_issues', 'leader_notes', 'instance_notes', + 'leader_milestones', 'instance_milestones', 'leader_boards', 'instance_boards', + 'leader_merge_requests', 'instance_merge_requests', 'leader_ci_pipelines', + 'instance_ci_pipelines', 'leader_environments', 'instance_environments', + 'leader_deployments', 'instance_deployments', 'leader_projects_prometheus_active', + 'instance_projects_prometheus_active', 'leader_service_desk_issues', + 'instance_service_desk_issues' + ) + ) + end +end diff --git a/app/services/users/refresh_authorized_projects_service.rb b/app/services/users/refresh_authorized_projects_service.rb index 8f6f5b937c4..3e07b811027 100644 --- a/app/services/users/refresh_authorized_projects_service.rb +++ b/app/services/users/refresh_authorized_projects_service.rb @@ -73,12 +73,11 @@ module Users # remove - The IDs of the authorization rows to remove. # add - Rows to insert in the form `[user id, project id, access level]` def update_authorizations(remove = [], add = []) - return if remove.empty? && add.empty? && user.authorized_projects_populated + return if remove.empty? && add.empty? User.transaction do user.remove_project_authorizations(remove) unless remove.empty? ProjectAuthorization.insert_authorizations(add) unless add.empty? - user.set_authorized_projects_column end # Since we batch insert authorization rows, Rails' associations may get @@ -101,38 +100,13 @@ module Users end def fresh_authorizations - ProjectAuthorization. - unscoped. - select('project_id, MAX(access_level) AS access_level'). - from("(#{project_authorizations_union.to_sql}) #{ProjectAuthorization.table_name}"). - group(:project_id) - end - - private - - # Returns a union query of projects that the user is authorized to access - def project_authorizations_union - relations = [ - # Personal projects - user.personal_projects.select("#{user.id} AS user_id, projects.id AS project_id, #{Gitlab::Access::MASTER} AS access_level"), - - # Projects the user is a member of - user.projects.select_for_project_authorization, - - # Projects of groups the user is a member of - user.groups_projects.select_for_project_authorization, - - # Projects of subgroups of groups the user is a member of - user.nested_groups_projects.select_for_project_authorization, - - # Projects shared with groups the user is a member of - user.groups.joins(:shared_projects).select_for_project_authorization, - - # Projects shared with subgroups of groups the user is a member of - user.nested_groups.joins(:shared_projects).select_for_project_authorization - ] + klass = if Group.supports_nested_groups? + Gitlab::ProjectAuthorizations::WithNestedGroups + else + Gitlab::ProjectAuthorizations::WithoutNestedGroups + end - Gitlab::SQL::Union.new(relations) + klass.new(user).calculate end end end diff --git a/app/services/web_hook_service.rb b/app/services/web_hook_service.rb new file mode 100644 index 00000000000..4241b912d5b --- /dev/null +++ b/app/services/web_hook_service.rb @@ -0,0 +1,120 @@ +class WebHookService + class InternalErrorResponse + attr_reader :body, :headers, :code + + def initialize + @headers = HTTParty::Response::Headers.new({}) + @body = '' + @code = 'internal error' + end + end + + include HTTParty + + # HTTParty timeout + default_timeout Gitlab.config.gitlab.webhook_timeout + + attr_accessor :hook, :data, :hook_name + + def initialize(hook, data, hook_name) + @hook = hook + @data = data + @hook_name = hook_name + end + + def execute + start_time = Time.now + + response = if parsed_url.userinfo.blank? + make_request(hook.url) + else + make_request_with_auth + end + + log_execution( + trigger: hook_name, + url: hook.url, + request_data: data, + response: response, + execution_duration: Time.now - start_time + ) + + [response.code, response.to_s] + rescue SocketError, OpenSSL::SSL::SSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Net::OpenTimeout => e + log_execution( + trigger: hook_name, + url: hook.url, + request_data: data, + response: InternalErrorResponse.new, + execution_duration: Time.now - start_time, + error_message: e.to_s + ) + + Rails.logger.error("WebHook Error => #{e}") + + [nil, e.to_s] + end + + def async_execute + Sidekiq::Client.enqueue(WebHookWorker, hook.id, data, hook_name) + end + + private + + def parsed_url + @parsed_url ||= URI.parse(hook.url) + end + + def make_request(url, basic_auth = false) + self.class.post(url, + body: data.to_json, + headers: build_headers(hook_name), + verify: hook.enable_ssl_verification, + basic_auth: basic_auth) + end + + def make_request_with_auth + post_url = hook.url.gsub("#{parsed_url.userinfo}@", '') + basic_auth = { + username: CGI.unescape(parsed_url.user), + password: CGI.unescape(parsed_url.password) + } + make_request(post_url, basic_auth) + end + + def log_execution(trigger:, url:, request_data:, response:, execution_duration:, error_message: nil) + # logging for ServiceHook's is not available + return if hook.is_a?(ServiceHook) + + WebHookLog.create( + web_hook: hook, + trigger: trigger, + url: url, + execution_duration: execution_duration, + request_headers: build_headers(hook_name), + request_data: request_data, + response_headers: format_response_headers(response), + response_body: response.body, + response_status: response.code, + internal_error_message: error_message + ) + end + + def build_headers(hook_name) + @headers ||= begin + { + 'Content-Type' => 'application/json', + 'X-Gitlab-Event' => hook_name.singularize.titleize + }.tap do |hash| + hash['X-Gitlab-Token'] = hook.token if hook.token.present? + end + end + end + + # Make response headers more stylish + # Net::HTTPHeader has downcased hash with arrays: { 'content-type' => ['text/html; charset=utf-8'] } + # This method format response to capitalized hash with strings: { 'Content-Type' => 'text/html; charset=utf-8' } + def format_response_headers(response) + response.headers.each_capitalized.to_h + end +end |