diff options
author | Shinya Maeda <shinya@gitlab.com> | 2018-07-06 08:38:24 +0300 |
---|---|---|
committer | Shinya Maeda <shinya@gitlab.com> | 2018-07-06 08:38:24 +0300 |
commit | 25bd5413200c9cd9368c753e2752f07735604547 (patch) | |
tree | c4cc75a53face6a6d0da8adca9a3994d3cc688b5 /lib | |
parent | b025e89e08888f3b393108f984a25c209526491b (diff) | |
parent | 969b7c565c6fe5cdfc54830d1da35f254ddaf530 (diff) |
Merge branch 'master' into build-chunks-on-object-storage
Diffstat (limited to 'lib')
28 files changed, 210 insertions, 134 deletions
diff --git a/lib/api/boards.rb b/lib/api/boards.rb index 6c706b2b4e1..086d39d5070 100644 --- a/lib/api/boards.rb +++ b/lib/api/boards.rb @@ -33,6 +33,7 @@ module API success Entities::Board end get '/:board_id' do + authorize!(:read_board, user_project) present board, with: Entities::Board end end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index bb48a86fe9e..3a6e707fd5b 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -769,28 +769,33 @@ module API class Todo < Grape::Entity expose :id - expose :project, using: Entities::BasicProjectDetails + expose :project, using: Entities::ProjectIdentity, if: -> (todo, _) { todo.project_id } + expose :group, using: 'API::Entities::NamespaceBasic', if: -> (todo, _) { todo.group_id } expose :author, using: Entities::UserBasic expose :action_name expose :target_type expose :target do |todo, options| - Entities.const_get(todo.target_type).represent(todo.target, options) + todo_target_class(todo.target_type).represent(todo.target, options) end expose :target_url do |todo, options| target_type = todo.target_type.underscore - target_url = "namespace_project_#{target_type}_url" + target_url = "#{todo.parent.class.to_s.underscore}_#{target_type}_url" target_anchor = "note_#{todo.note_id}" if todo.note_id? Gitlab::Routing .url_helpers - .public_send(target_url, todo.project.namespace, todo.project, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend + .public_send(target_url, todo.parent, todo.target, anchor: target_anchor) # rubocop:disable GitlabSecurity/PublicSend end expose :body expose :state expose :created_at + + def todo_target_class(target_type) + ::API::Entities.const_get(target_type) + end end class NamespaceBasic < Grape::Entity @@ -1010,7 +1015,7 @@ module API expose :description expose :ip_address expose :active - expose :is_shared + expose :instance_type?, as: :is_shared expose :name expose :online?, as: :online expose :status @@ -1024,7 +1029,7 @@ module API expose :access_level expose :version, :revision, :platform, :architecture expose :contacted_at - expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.is_shared? } + expose :token, if: lambda { |runner, options| options[:current_user].admin? || !runner.instance_type? } expose :projects, with: Entities::BasicProjectDetails do |runner, options| if options[:current_user].admin? runner.projects @@ -1198,6 +1203,7 @@ module API class RunnerInfo < Grape::Entity expose :metadata_timeout, as: :timeout + expose :runner_session_url end class Step < Grape::Entity diff --git a/lib/api/pipelines.rb b/lib/api/pipelines.rb index 8374a57edfa..5d33a13d035 100644 --- a/lib/api/pipelines.rb +++ b/lib/api/pipelines.rb @@ -31,7 +31,7 @@ module API get ':id/pipelines' do authorize! :read_pipeline, user_project - pipelines = PipelinesFinder.new(user_project, params).execute + pipelines = PipelinesFinder.new(user_project, current_user, params).execute present paginate(pipelines), with: Entities::PipelineBasic end diff --git a/lib/api/runner.rb b/lib/api/runner.rb index 96a02914faa..d0cc0945a5f 100644 --- a/lib/api/runner.rb +++ b/lib/api/runner.rb @@ -24,13 +24,13 @@ module API attributes = if runner_registration_token_valid? # Create shared runner. Requires admin access - attributes.merge(is_shared: true, runner_type: :instance_type) + attributes.merge(runner_type: :instance_type) elsif project = Project.find_by(runners_token: params[:token]) # Create a specific runner for the project - attributes.merge(is_shared: false, runner_type: :project_type, projects: [project]) + attributes.merge(runner_type: :project_type, projects: [project]) elsif group = Group.find_by(runners_token: params[:token]) # Create a specific runner for the group - attributes.merge(is_shared: false, runner_type: :group_type, groups: [group]) + attributes.merge(runner_type: :group_type, groups: [group]) else forbidden! end @@ -81,6 +81,11 @@ module API requires :token, type: String, desc: %q(Runner's authentication token) optional :last_update, type: String, desc: %q(Runner's queue last_update token) optional :info, type: Hash, desc: %q(Runner's metadata) + optional :session, type: Hash, desc: %q(Runner's session data) do + optional :url, type: String, desc: %q(Session's url) + optional :certificate, type: String, desc: %q(Session's certificate) + optional :authorization, type: String, desc: %q(Session's authorization) + end end post '/request' do authenticate_runner! @@ -90,14 +95,16 @@ module API break no_content! end - if current_runner.runner_queue_value_latest?(params[:last_update]) - header 'X-GitLab-Last-Update', params[:last_update] + runner_params = declared_params(include_missing: false) + + if current_runner.runner_queue_value_latest?(runner_params[:last_update]) + header 'X-GitLab-Last-Update', runner_params[:last_update] Gitlab::Metrics.add_event(:build_not_found_cached) break no_content! end new_update = current_runner.ensure_runner_queue_value - result = ::Ci::RegisterJobService.new(current_runner).execute + result = ::Ci::RegisterJobService.new(current_runner).execute(runner_params) if result.valid? if result.build diff --git a/lib/api/runners.rb b/lib/api/runners.rb index 2b78075ddbf..2071c5a62c1 100644 --- a/lib/api/runners.rb +++ b/lib/api/runners.rb @@ -119,7 +119,7 @@ module API use :pagination end get ':id/runners' do - runners = filter_runners(Ci::Runner.owned_or_shared(user_project.id), params[:scope]) + runners = filter_runners(Ci::Runner.owned_or_instance_wide(user_project.id), params[:scope]) present paginate(runners), with: Entities::Runner end @@ -170,6 +170,11 @@ module API render_api_error!('Scope contains invalid value', 400) end + # Support deprecated scopes + if runners.respond_to?("deprecated_#{scope}") + scope = "deprecated_#{scope}" + end + runners.public_send(scope) # rubocop:disable GitlabSecurity/PublicSend end @@ -180,7 +185,7 @@ module API end def authenticate_show_runner!(runner) - return if runner.is_shared || current_user.admin? + return if runner.instance_type? || current_user.admin? forbidden!("No access granted") unless can?(current_user, :read_runner, runner) end diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index afc2ca4e362..8275bb9e149 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -4,27 +4,20 @@ module Banzai # # Extends HTML::Pipeline::SanitizationFilter with a custom whitelist. class SanitizationFilter < HTML::Pipeline::SanitizationFilter + include Gitlab::Utils::StrongMemoize + UNSAFE_PROTOCOLS = %w(data javascript vbscript).freeze TABLE_ALIGNMENT_PATTERN = /text-align: (?<alignment>center|left|right)/ def whitelist - whitelist = super - - customize_whitelist(whitelist) - - whitelist + strong_memoize(:whitelist) do + customize_whitelist(super.dup) + end end private - def customized?(transformers) - transformers.last.source_location[0] == __FILE__ - end - def customize_whitelist(whitelist) - # Only push these customizations once - return if customized?(whitelist[:transformers]) - # Allow table alignment; we whitelist specific text-align values in a # transformer below whitelist[:attributes]['th'] = %w(style) diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 1e283cc092b..eb246d393a1 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -55,7 +55,7 @@ module Gitlab id: runner.id, description: runner.description, active: runner.active?, - is_shared: runner.is_shared? + is_shared: runner.instance_type? } end end diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index a9e209d5b71..d16a55720b7 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -247,6 +247,7 @@ module Gitlab lines = highlighted_diff_lines return if lines.empty? + return if blob.nil? last_line = lines.last diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index c358ae428cf..be25e1bab21 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -34,7 +34,7 @@ module Gitlab end def cache_key - [@merge_request_diff, 'highlighted-diff-files', diff_options] + [@merge_request_diff, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options] end private diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 2b3ebfbb9ff..1faf7770634 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -1,6 +1,8 @@ module Gitlab module Diff class Line + SERIALIZE_KEYS = %i(line_code text type index old_pos new_pos).freeze + attr_reader :line_code, :type, :index, :old_pos, :new_pos attr_writer :rich_text attr_accessor :text @@ -19,13 +21,9 @@ module Gitlab new(hash[:text], hash[:type], hash[:index], hash[:old_pos], hash[:new_pos], line_code: hash[:line_code]) end - def serialize_keys - @serialize_keys ||= %i(line_code text type index old_pos new_pos) - end - def to_hash hash = {} - serialize_keys.each { |key| hash[key] = send(key) } # rubocop:disable GitlabSecurity/PublicSend + SERIALIZE_KEYS.each { |key| hash[key] = send(key) } # rubocop:disable GitlabSecurity/PublicSend hash end diff --git a/lib/gitlab/git/commit_stats.rb b/lib/gitlab/git/commit_stats.rb index 8463b1eb794..ae6f554bc06 100644 --- a/lib/gitlab/git/commit_stats.rb +++ b/lib/gitlab/git/commit_stats.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: 1 RPC, migration in progress. - # Gitlab::Git::CommitStats counts the additions, deletions, and total changes # in a commit. module Gitlab @@ -16,12 +14,8 @@ module Gitlab @deletions = 0 @total = 0 - repo.gitaly_migrate(:commit_stats) do |is_enabled| - if is_enabled - gitaly_stats(repo, commit) - else - rugged_stats(commit) - end + repo.wrapped_gitaly_errors do + gitaly_stats(repo, commit) end end @@ -31,12 +25,6 @@ module Gitlab @deletions = stats.deletions @total = @additions + @deletions end - - def rugged_stats(commit) - diff = commit.rugged_diff_from_parent - _files_changed, @additions, @deletions = diff.stat - @total = @additions + @deletions - end end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 706aa7343be..bbfe6ab1d95 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -251,7 +251,6 @@ module Gitlab # Returns an Array of Tags # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/390 def tags wrapped_gitaly_errors do gitaly_ref_client.tags @@ -602,17 +601,9 @@ module Gitlab # @repository.submodule_url_for('master', 'rack') # # => git@localhost:rack.git # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/329 def submodule_url_for(ref, path) - Gitlab::GitalyClient.migrate(:submodule_url_for) do |is_enabled| - if is_enabled - gitaly_submodule_url_for(ref, path) - else - if submodules(ref).any? - submodule = submodules(ref)[path] - submodule['url'] if submodule - end - end + wrapped_gitaly_errors do + gitaly_submodule_url_for(ref, path) end end @@ -837,22 +828,14 @@ module Gitlab # Ex. # repo.ls_files('master') # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/327 def ls_files(ref) gitaly_commit_client.ls_files(ref) end - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/328 def copy_gitattributes(ref) - Gitlab::GitalyClient.migrate(:apply_gitattributes) do |is_enabled| - if is_enabled - gitaly_copy_gitattributes(ref) - else - rugged_copy_gitattributes(ref) - end + wrapped_gitaly_errors do + gitaly_repository_client.apply_gitattributes(ref) end - rescue GRPC::InvalidArgument - raise InvalidRef end def info_attributes diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index b6ceb542dd1..cb851b76a23 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -1,5 +1,3 @@ -# Gitaly note: JV: needs 1 RPC, migration is in progress. - module Gitlab module Git class Tree @@ -17,12 +15,8 @@ module Gitlab def where(repository, sha, path = nil, recursive = false) path = nil if path == '' || path == '/' - Gitlab::GitalyClient.migrate(:tree_entries) do |is_enabled| - if is_enabled - repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive) - else - tree_entries_from_rugged(repository, sha, path, recursive) - end + repository.wrapped_gitaly_errors do + repository.gitaly_commit_client.tree_entries(repository, sha, path, recursive) end end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 620362b52a9..66e781a8e5b 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -429,7 +429,7 @@ module Gitlab def self.count_stack return unless RequestStore.active? - stack_string = caller.drop(1).join("\n") + stack_string = Gitlab::Profiler.clean_backtrace(caller).drop(1).join("\n") RequestStore.store[:stack_counter] ||= Hash.new diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 28554208984..1840bf45154 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -13,7 +13,7 @@ module Gitlab oid: oid, limit: limit ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request, timeout: GitalyClient.fast_timeout) data = '' blob = nil @@ -43,7 +43,7 @@ module Gitlab blob_ids: blob_ids ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request, timeout: GitalyClient.medium_timeout) map_lfs_pointers(response) end @@ -66,7 +66,7 @@ module Gitlab :blob_service, :get_blobs, request, - timeout: GitalyClient.default_timeout + timeout: GitalyClient.fast_timeout ) GitalyClient::BlobsStitcher.new(response) @@ -85,7 +85,7 @@ module Gitlab request.not_in_refs += not_in end - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_new_lfs_pointers, request) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_new_lfs_pointers, request, timeout: GitalyClient.medium_timeout) map_lfs_pointers(response) end @@ -96,7 +96,7 @@ module Gitlab revision: encode_binary(revision) ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request) + response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_all_lfs_pointers, request, timeout: GitalyClient.medium_timeout) map_lfs_pointers(response) end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index d979ba0eb14..72e1e59d8df 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -70,7 +70,7 @@ module Gitlab def commit_deltas(commit) request = Gitaly::CommitDeltaRequest.new(diff_from_parent_request_params(commit)) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request) + response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request, timeout: GitalyClient.fast_timeout) response.flat_map { |msg| msg.deltas } end @@ -302,7 +302,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum) + response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum, timeout: GitalyClient.fast_timeout) response.flat_map do |msg| msg.shas.map { |sha| EncodingHelper.encode!(sha) } @@ -330,7 +330,7 @@ module Gitlab def get_commit_signatures(commit_ids) request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) - response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request) + response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout) signatures = Hash.new { |h, k| h[k] = [''.b, ''.b] } current_commit_id = nil @@ -349,7 +349,7 @@ module Gitlab def get_commit_messages(commit_ids) request = Gitaly::GetCommitMessagesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) - response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request) + response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request, timeout: GitalyClient.fast_timeout) messages = Hash.new { |h, k| h[k] = ''.b } current_commit_id = nil diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index e14734495a8..b1a01b185e6 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -46,7 +46,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage) + response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage, timeout: GitalyClient.medium_timeout) if response.resolution_error.present? raise Gitlab::Git::Conflict::Resolver::ResolutionError, response.resolution_error diff --git a/lib/gitlab/gitaly_client/namespace_service.rb b/lib/gitlab/gitaly_client/namespace_service.rb index bd7c345ac01..d4e982b649a 100644 --- a/lib/gitlab/gitaly_client/namespace_service.rb +++ b/lib/gitlab/gitaly_client/namespace_service.rb @@ -8,31 +8,31 @@ module Gitlab def exists?(name) request = Gitaly::NamespaceExistsRequest.new(storage_name: @storage, name: name) - gitaly_client_call(:namespace_exists, request).exists + gitaly_client_call(:namespace_exists, request, timeout: GitalyClient.fast_timeout).exists end def add(name) request = Gitaly::AddNamespaceRequest.new(storage_name: @storage, name: name) - gitaly_client_call(:add_namespace, request) + gitaly_client_call(:add_namespace, request, timeout: GitalyClient.fast_timeout) end def remove(name) request = Gitaly::RemoveNamespaceRequest.new(storage_name: @storage, name: name) - gitaly_client_call(:remove_namespace, request) + gitaly_client_call(:remove_namespace, request, timeout: nil) end def rename(from, to) request = Gitaly::RenameNamespaceRequest.new(storage_name: @storage, from: from, to: to) - gitaly_client_call(:rename_namespace, request) + gitaly_client_call(:rename_namespace, request, timeout: GitalyClient.fast_timeout) end private - def gitaly_client_call(type, request) - GitalyClient.call(@storage, :namespace_service, type, request) + def gitaly_client_call(type, request, timeout: nil) + GitalyClient.call(@storage, :namespace_service, type, request, timeout: timeout) end end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index c04183a348f..ab2c61f6782 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -17,7 +17,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request) + response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.medium_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -33,7 +33,7 @@ module Gitlab message: encode_binary(message.to_s) ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request) + response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.medium_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error elsif response.exists @@ -276,7 +276,8 @@ module Gitlab :operation_service, :"user_#{rpc}", request, - remote_storage: start_repository.storage + remote_storage: start_repository.storage, + timeout: GitalyClient.medium_timeout ) handle_cherry_pick_or_revert_response(response) diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 3ac46be6208..7f4eed9222a 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -12,7 +12,7 @@ module Gitlab def branches request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout) consume_find_all_branches_response(response) end @@ -23,26 +23,26 @@ module Gitlab merged_only: true, merged_branches: branch_names.map { |s| encode_binary(s) } ) - response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout) consume_find_all_branches_response(response) end def default_branch_name request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request) + response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request, timeout: GitalyClient.fast_timeout) Gitlab::Git.branch_name(response.name) end def branch_names request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request, timeout: GitalyClient.fast_timeout) consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) } end def tag_names request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request, timeout: GitalyClient.fast_timeout) consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } end @@ -67,19 +67,19 @@ module Gitlab def local_branches(sort_by: nil) request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo) request.sort_by = sort_by_param(sort_by) if sort_by - response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request) + response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request, timeout: GitalyClient.fast_timeout) consume_find_local_branches_response(response) end def tags request = Gitaly::FindAllTagsRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request) + response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request, timeout: GitalyClient.medium_timeout) consume_tags_response(response) end def ref_exists?(ref_name) request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: encode_binary(ref_name)) - response = GitalyClient.call(@storage, :ref_service, :ref_exists, request) + response = GitalyClient.call(@storage, :ref_service, :ref_exists, request, timeout: GitalyClient.fast_timeout) response.value rescue GRPC::InvalidArgument => e raise ArgumentError, e.message @@ -91,7 +91,7 @@ module Gitlab name: encode_binary(branch_name) ) - response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request) + response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request, timeout: GitalyClient.medium_timeout) branch = response.branch return unless branch @@ -140,7 +140,7 @@ module Gitlab except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) } ) - response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request) + response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.fast_timeout) raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? end @@ -153,7 +153,7 @@ module Gitlab limit: limit ) - stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request) + stream = GitalyClient.call(@repository.storage, :ref_service, :list_tag_names_containing_commit, request, timeout: GitalyClient.medium_timeout) consume_ref_contains_sha_response(stream, :tag_names) end @@ -166,14 +166,14 @@ module Gitlab limit: limit ) - stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request) + stream = GitalyClient.call(@repository.storage, :ref_service, :list_branch_names_containing_commit, request, timeout: GitalyClient.medium_timeout) consume_ref_contains_sha_response(stream, :branch_names) end def get_tag_messages(tag_ids) request = Gitaly::GetTagMessagesRequest.new(repository: @gitaly_repo, tag_ids: tag_ids) - response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_messages, request) + response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_messages, request, timeout: GitalyClient.fast_timeout) messages = Hash.new { |h, k| h[k] = ''.b } current_tag_id = nil diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index f2d699d9dfb..1381e033d4b 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -28,13 +28,13 @@ module Gitlab mirror_refmaps: Array.wrap(mirror_refmaps).map(&:to_s) ) - GitalyClient.call(@storage, :remote_service, :add_remote, request) + GitalyClient.call(@storage, :remote_service, :add_remote, request, timeout: GitalyClient.fast_timeout) end def remove_remote(name) request = Gitaly::RemoveRemoteRequest.new(repository: @gitaly_repo, name: name) - response = GitalyClient.call(@storage, :remote_service, :remove_remote, request) + response = GitalyClient.call(@storage, :remote_service, :remove_remote, request, timeout: GitalyClient.fast_timeout) response.result end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index ca986434221..982f8d0963b 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -21,7 +21,7 @@ module Gitlab def cleanup request = Gitaly::CleanupRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :cleanup, request) + GitalyClient.call(@storage, :repository_service, :cleanup, request, timeout: GitalyClient.fast_timeout) end def garbage_collect(create_bitmap) @@ -41,19 +41,21 @@ module Gitlab def repository_size request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :repository_size, request) + response = GitalyClient.call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.medium_timeout) response.size end def apply_gitattributes(revision) request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: encode_binary(revision)) - GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request) + GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request, timeout: GitalyClient.fast_timeout) + rescue GRPC::InvalidArgument => ex + raise Gitlab::Git::Repository::InvalidRef, ex end def info_attributes request = Gitaly::GetInfoAttributesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :get_info_attributes, request) + response = GitalyClient.call(@storage, :repository_service, :get_info_attributes, request, timeout: GitalyClient.fast_timeout) response.each_with_object("") do |message, attributes| attributes << message.attributes end @@ -80,7 +82,7 @@ module Gitlab def create_repository request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :create_repository, request) + GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.medium_timeout) end def has_local_branches? @@ -96,7 +98,7 @@ module Gitlab revisions: revisions.map { |r| encode_binary(r) } ) - response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request) + response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request, timeout: GitalyClient.fast_timeout) response.base.presence end @@ -256,7 +258,7 @@ module Gitlab ) request.old_revision = old_ref.b unless old_ref.nil? - response = GitalyClient.call(@storage, :repository_service, :write_ref, request) + response = GitalyClient.call(@storage, :repository_service, :write_ref, request, timeout: GitalyClient.fast_timeout) raise Gitlab::Git::CommandError, encode!(response.error) if response.error.present? @@ -286,7 +288,7 @@ module Gitlab def calculate_checksum request = Gitaly::CalculateChecksumRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request) + response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request, timeout: GitalyClient.fast_timeout) response.checksum.presence rescue GRPC::DataLoss => e raise Gitlab::Git::Repository::InvalidRepository.new(e) @@ -295,12 +297,12 @@ module Gitlab def raw_changes_between(from, to) request = Gitaly::GetRawChangesRequest.new(repository: @gitaly_repo, from_revision: from, to_revision: to) - GitalyClient.call(@storage, :repository_service, :get_raw_changes, request) + GitalyClient.call(@storage, :repository_service, :get_raw_changes, request, timeout: GitalyClient.fast_timeout) end def search_files_by_name(ref, query) request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query) - GitalyClient.call(@storage, :repository_service, :search_files_by_name, request).flat_map(&:files) + GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end def search_files_by_content(ref, query) diff --git a/lib/gitlab/gitaly_client/wiki_service.rb b/lib/gitlab/gitaly_client/wiki_service.rb index 2dfe055a496..6cb049c1f68 100644 --- a/lib/gitlab/gitaly_client/wiki_service.rb +++ b/lib/gitlab/gitaly_client/wiki_service.rb @@ -69,7 +69,7 @@ module Gitlab commit_details: gitaly_commit_details(commit_details) ) - GitalyClient.call(@repository.storage, :wiki_service, :wiki_delete_page, request) + GitalyClient.call(@repository.storage, :wiki_service, :wiki_delete_page, request, timeout: GitalyClient.medium_timeout) end def find_page(title:, version: nil, dir: nil) @@ -80,14 +80,14 @@ module Gitlab directory: encode_binary(dir) ) - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_page, request, timeout: GitalyClient.fast_timeout) wiki_page_from_iterator(response) end def get_all_pages request = Gitaly::WikiGetAllPagesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_all_pages, request, timeout: GitalyClient.medium_timeout) pages = [] loop do @@ -113,7 +113,7 @@ module Gitlab per_page: options[:per_page] || Gollum::Page.per_page ) - stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request) + stream = GitalyClient.call(@repository.storage, :wiki_service, :wiki_get_page_versions, request, timeout: GitalyClient.medium_timeout) versions = [] stream.each do |message| @@ -132,7 +132,7 @@ module Gitlab revision: encode_binary(revision) ) - response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request) + response = GitalyClient.call(@repository.storage, :wiki_service, :wiki_find_file, request, timeout: GitalyClient.fast_timeout) wiki_file = nil response.each do |message| diff --git a/lib/gitlab/graphql/connections.rb b/lib/gitlab/graphql/connections.rb new file mode 100644 index 00000000000..2582ffeb2a8 --- /dev/null +++ b/lib/gitlab/graphql/connections.rb @@ -0,0 +1,12 @@ +module Gitlab + module Graphql + module Connections + def self.use(_schema) + GraphQL::Relay::BaseConnection.register_connection_implementation( + ActiveRecord::Relation, + Gitlab::Graphql::Connections::KeysetConnection + ) + end + end + end +end diff --git a/lib/gitlab/graphql/connections/keyset_connection.rb b/lib/gitlab/graphql/connections/keyset_connection.rb new file mode 100644 index 00000000000..abee2afe144 --- /dev/null +++ b/lib/gitlab/graphql/connections/keyset_connection.rb @@ -0,0 +1,73 @@ +module Gitlab + module Graphql + module Connections + class KeysetConnection < GraphQL::Relay::BaseConnection + def cursor_from_node(node) + encode(node[order_field].to_s) + end + + def sliced_nodes + @sliced_nodes ||= + begin + sliced = nodes + + sliced = sliced.where(before_slice) if before.present? + sliced = sliced.where(after_slice) if after.present? + + sliced + end + end + + def paged_nodes + if first && last + raise Gitlab::Graphql::Errors::ArgumentError.new("Can only provide either `first` or `last`, not both") + end + + if last + sliced_nodes.last(limit_value) + else + sliced_nodes.limit(limit_value) + end + end + + private + + def before_slice + if sort_direction == :asc + table[order_field].lt(decode(before)) + else + table[order_field].gt(decode(before)) + end + end + + def after_slice + if sort_direction == :asc + table[order_field].gt(decode(after)) + else + table[order_field].lt(decode(after)) + end + end + + def limit_value + @limit_value ||= [first, last, max_page_size].compact.min + end + + def table + nodes.arel_table + end + + def order_info + @order_info ||= nodes.order_values.first + end + + def order_field + @order_field ||= order_info&.expr&.name || nodes.primary_key + end + + def sort_direction + @order_direction ||= order_info&.direction || :desc + end + end + end + end +end diff --git a/lib/gitlab/graphql/errors.rb b/lib/gitlab/graphql/errors.rb new file mode 100644 index 00000000000..1d8e8307ab9 --- /dev/null +++ b/lib/gitlab/graphql/errors.rb @@ -0,0 +1,8 @@ +module Gitlab + module Graphql + module Errors + BaseError = Class.new(GraphQL::ExecutionError) + ArgumentError = Class.new(BaseError) + end + end +end diff --git a/lib/gitlab/graphql/present/instrumentation.rb b/lib/gitlab/graphql/present/instrumentation.rb index 6f2b26c9676..f87fd147b15 100644 --- a/lib/gitlab/graphql/present/instrumentation.rb +++ b/lib/gitlab/graphql/present/instrumentation.rb @@ -3,6 +3,8 @@ module Gitlab module Present class Instrumentation def instrument(type, field) + return field unless field.metadata[:type_class] + presented_in = field.metadata[:type_class].owner return field unless presented_in.respond_to?(:presenter_class) return field unless presented_in.presenter_class diff --git a/lib/gitlab/middleware/multipart.rb b/lib/gitlab/middleware/multipart.rb index a5f5d719cc1..9753be6d5c3 100644 --- a/lib/gitlab/middleware/multipart.rb +++ b/lib/gitlab/middleware/multipart.rb @@ -42,10 +42,10 @@ module Gitlab key, value = parsed_field.first if value.nil? - value = open_file(tmp_path, @request.params["#{key}.name"]) + value = open_file(@request.params, key) @open_files << value else - value = decorate_params_value(value, @request.params[key], tmp_path) + value = decorate_params_value(value, @request.params[key]) end @request.update_param(key, value) @@ -57,7 +57,7 @@ module Gitlab end # This function calls itself recursively - def decorate_params_value(path_hash, value_hash, tmp_path) + def decorate_params_value(path_hash, value_hash) unless path_hash.is_a?(Hash) && path_hash.count == 1 raise "invalid path: #{path_hash.inspect}" end @@ -70,19 +70,21 @@ module Gitlab case path_value when nil - value_hash[path_key] = open_file(tmp_path, value_hash.dig(path_key, '.name')) + value_hash[path_key] = open_file(value_hash.dig(path_key), '') @open_files << value_hash[path_key] value_hash when Hash - decorate_params_value(path_value, value_hash[path_key], tmp_path) + decorate_params_value(path_value, value_hash[path_key]) value_hash else raise "unexpected path value: #{path_value.inspect}" end end - def open_file(path, name) - ::UploadedFile.new(path, filename: name || File.basename(path), content_type: 'application/octet-stream') + def open_file(params, key) + ::UploadedFile.from_params( + params, key, + Gitlab.config.uploads.storage_path) end end |