From 6b3f0a47a2410b5a2a9fc1e78ff2d006b05a3e05 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 30 Jun 2021 11:40:06 +0000 Subject: Add latest changes from gitlab-org/security/gitlab@14-0-stable-ee --- .../diff_viewer/viewers/not_diffable.vue | 2 +- .../integrations/slack_mattermost_notifier.rb | 2 +- app/models/integrations/bamboo.rb | 1 + app/models/integrations/base_issue_tracker.rb | 2 +- app/models/integrations/drone_ci.rb | 7 ++- app/models/integrations/external_wiki.rb | 2 +- app/models/integrations/mock_ci.rb | 2 +- app/models/integrations/teamcity.rb | 5 +- app/models/integrations/unify_circuit.rb | 3 +- app/models/integrations/webex_teams.rb | 2 +- app/models/protected_branch/push_access_level.rb | 2 +- app/services/web_hook_service.rb | 3 +- .../projects/diffs/viewers/_not_diffable.html.haml | 2 +- lib/gitlab/diff/file.rb | 11 +++- lib/gitlab/diff/parser.rb | 2 +- lib/gitlab/git/diff.rb | 11 +++- lib/gitlab/http.rb | 18 ++++++- locale/gitlab.pot | 6 +-- spec/features/expand_collapse_diffs_spec.rb | 2 +- spec/features/projects/diffs/diff_show_spec.rb | 10 ++++ .../file_collection/merge_request_diff_spec.rb | 2 +- spec/lib/gitlab/diff/file_spec.rb | 58 +++++++++++++++++----- spec/lib/gitlab/diff/parser_spec.rb | 10 ++++ spec/lib/gitlab/http_spec.rb | 41 +++++++++++++++ .../protected_branch/push_access_level_spec.rb | 2 +- 25 files changed, 169 insertions(+), 39 deletions(-) diff --git a/app/assets/javascripts/vue_shared/components/diff_viewer/viewers/not_diffable.vue b/app/assets/javascripts/vue_shared/components/diff_viewer/viewers/not_diffable.vue index d4d3038f066..5a6b1c19027 100644 --- a/app/assets/javascripts/vue_shared/components/diff_viewer/viewers/not_diffable.vue +++ b/app/assets/javascripts/vue_shared/components/diff_viewer/viewers/not_diffable.vue @@ -1,5 +1,5 @@ diff --git a/app/models/concerns/integrations/slack_mattermost_notifier.rb b/app/models/concerns/integrations/slack_mattermost_notifier.rb index a919fc840fd..cb6fafa8de0 100644 --- a/app/models/concerns/integrations/slack_mattermost_notifier.rb +++ b/app/models/concerns/integrations/slack_mattermost_notifier.rb @@ -17,7 +17,7 @@ module Integrations class HTTPClient def self.post(uri, params = {}) params.delete(:http_options) # these are internal to the client and we do not want them - Gitlab::HTTP.post(uri, body: params) + Gitlab::HTTP.post(uri, body: params, use_read_total_timeout: true) end end end diff --git a/app/models/integrations/bamboo.rb b/app/models/integrations/bamboo.rb index dbd7aedf4fe..fef2774c593 100644 --- a/app/models/integrations/bamboo.rb +++ b/app/models/integrations/bamboo.rb @@ -173,6 +173,7 @@ module Integrations query_params[:os_authType] = 'basic' params[:basic_auth] = basic_auth + params[:use_read_total_timeout] = true params end diff --git a/app/models/integrations/base_issue_tracker.rb b/app/models/integrations/base_issue_tracker.rb index 6c24f762cd5..3fd67205e92 100644 --- a/app/models/integrations/base_issue_tracker.rb +++ b/app/models/integrations/base_issue_tracker.rb @@ -107,7 +107,7 @@ module Integrations result = false begin - response = Gitlab::HTTP.head(self.project_url, verify: true) + response = Gitlab::HTTP.head(self.project_url, verify: true, use_read_total_timeout: true) if response message = "#{self.type} received response #{response.code} when attempting to connect to #{self.project_url}" diff --git a/app/models/integrations/drone_ci.rb b/app/models/integrations/drone_ci.rb index 096f7093b8c..0f021356815 100644 --- a/app/models/integrations/drone_ci.rb +++ b/app/models/integrations/drone_ci.rb @@ -51,9 +51,12 @@ module Integrations end def calculate_reactive_cache(sha, ref) - response = Gitlab::HTTP.try_get(commit_status_path(sha, ref), + response = Gitlab::HTTP.try_get( + commit_status_path(sha, ref), verify: enable_ssl_verification, - extra_log_info: { project_id: project_id }) + extra_log_info: { project_id: project_id }, + use_read_total_timeout: true + ) status = if response && response.code == 200 && response['status'] diff --git a/app/models/integrations/external_wiki.rb b/app/models/integrations/external_wiki.rb index fec435443fa..2a8d598117b 100644 --- a/app/models/integrations/external_wiki.rb +++ b/app/models/integrations/external_wiki.rb @@ -39,7 +39,7 @@ module Integrations end def execute(_data) - response = Gitlab::HTTP.get(properties['external_wiki_url'], verify: true) + response = Gitlab::HTTP.get(properties['external_wiki_url'], verify: true, use_read_total_timeout: true) response.body if response.code == 200 rescue StandardError nil diff --git a/app/models/integrations/mock_ci.rb b/app/models/integrations/mock_ci.rb index d31f6381767..a0eae9e4abf 100644 --- a/app/models/integrations/mock_ci.rb +++ b/app/models/integrations/mock_ci.rb @@ -55,7 +55,7 @@ module Integrations # # => 'running' # def commit_status(sha, ref) - response = Gitlab::HTTP.get(commit_status_path(sha), verify: false) + response = Gitlab::HTTP.get(commit_status_path(sha), verify: false, use_read_total_timeout: true) read_commit_status(response) rescue Errno::ECONNREFUSED :error diff --git a/app/models/integrations/teamcity.rb b/app/models/integrations/teamcity.rb index 8284d5963ae..3f14c5d82b3 100644 --- a/app/models/integrations/teamcity.rb +++ b/app/models/integrations/teamcity.rb @@ -170,7 +170,7 @@ module Integrations end def get_path(path) - Gitlab::HTTP.try_get(build_url(path), verify: false, basic_auth: basic_auth, extra_log_info: { project_id: project_id }) + Gitlab::HTTP.try_get(build_url(path), verify: false, basic_auth: basic_auth, extra_log_info: { project_id: project_id }, use_read_total_timeout: true) end def post_to_build_queue(data, branch) @@ -180,7 +180,8 @@ module Integrations ""\ '', headers: { 'Content-type' => 'application/xml' }, - basic_auth: basic_auth + basic_auth: basic_auth, + use_read_total_timeout: true ) end diff --git a/app/models/integrations/unify_circuit.rb b/app/models/integrations/unify_circuit.rb index 03363c7c8b0..834222834e9 100644 --- a/app/models/integrations/unify_circuit.rb +++ b/app/models/integrations/unify_circuit.rb @@ -49,7 +49,8 @@ module Integrations response = Gitlab::HTTP.post(webhook, body: { subject: message.project_name, text: message.summary, - markdown: true + markdown: true, + use_read_total_timeout: true }.to_json) response if response.success? diff --git a/app/models/integrations/webex_teams.rb b/app/models/integrations/webex_teams.rb index 3f420331035..6fd82a32035 100644 --- a/app/models/integrations/webex_teams.rb +++ b/app/models/integrations/webex_teams.rb @@ -44,7 +44,7 @@ module Integrations def notify(message, opts) header = { 'Content-Type' => 'application/json' } - response = Gitlab::HTTP.post(webhook, headers: header, body: { markdown: message.summary }.to_json) + response = Gitlab::HTTP.post(webhook, headers: header, body: { markdown: message.summary }.to_json, use_read_total_timeout: true) response if response.success? end diff --git a/app/models/protected_branch/push_access_level.rb b/app/models/protected_branch/push_access_level.rb index ea51dca8a42..5248834a2f2 100644 --- a/app/models/protected_branch/push_access_level.rb +++ b/app/models/protected_branch/push_access_level.rb @@ -20,7 +20,7 @@ class ProtectedBranch::PushAccessLevel < ApplicationRecord def check_access(user) if user && deploy_key.present? - return true if user.can?(:read_project, project) && enabled_deploy_key_for_user?(deploy_key, user) + return user.can?(:read_project, project) && enabled_deploy_key_for_user?(deploy_key, user) end super diff --git a/app/services/web_hook_service.rb b/app/services/web_hook_service.rb index 77d2139b3d1..1d5b38575bb 100644 --- a/app/services/web_hook_service.rb +++ b/app/services/web_hook_service.rb @@ -42,6 +42,7 @@ class WebHookService @uniqueness_token = uniqueness_token @request_options = { timeout: Gitlab.config.gitlab.webhook_timeout, + use_read_total_timeout: true, allow_local_requests: hook.allow_local_requests? } end @@ -68,7 +69,7 @@ class WebHookService { status: :success, http_status: response.code, - message: response.to_s + message: response.body } rescue *Gitlab::HTTP::HTTP_ERRORS, Gitlab::Json::LimitedEncoder::LimitExceeded, URI::InvalidURIError => e diff --git a/app/views/projects/diffs/viewers/_not_diffable.html.haml b/app/views/projects/diffs/viewers/_not_diffable.html.haml index 7c55e272f56..63034331f6a 100644 --- a/app/views/projects/diffs/viewers/_not_diffable.html.haml +++ b/app/views/projects/diffs/viewers/_not_diffable.html.haml @@ -1,2 +1,2 @@ .nothing-here-block - = _("This diff was suppressed by a .gitattributes entry.") + = _("File suppressed by a .gitattributes entry or the file's encoding is unsupported.") diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index dcd4bbdabf5..35581952f4a 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -250,7 +250,7 @@ module Gitlab end def diffable? - repository.attributes(file_path).fetch('diff') { true } + diffable_by_attribute? && !text_with_binary_notice? end def binary_in_repo? @@ -366,6 +366,15 @@ module Gitlab private + def diffable_by_attribute? + repository.attributes(file_path).fetch('diff') { true } + end + + # NOTE: Files with unsupported encodings (e.g. UTF-16) are treated as binary by git, but they are recognized as text files during encoding detection. These files have `Binary files a/filename and b/filename differ' as their raw diff content which cannot be used. We need to handle this special case and avoid displaying incorrect diff. + def text_with_binary_notice? + text? && has_binary_notice? + end + def fetch_blob(sha, path) return unless sha diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index 4a47e4b80b6..adb711ca89f 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -6,7 +6,7 @@ module Gitlab include Enumerable def parse(lines, diff_file: nil) - return [] if lines.blank? + return [] if lines.blank? || Git::Diff.has_binary_notice?(lines.first) @lines = lines line_obj_index = 0 diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 53df0b7b389..8325eadce2f 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -33,6 +33,8 @@ module Gitlab SERIALIZE_KEYS = %i(diff new_path old_path a_mode b_mode new_file renamed_file deleted_file too_large).freeze + BINARY_NOTICE_PATTERN = %r(Binary files a\/(.*) and b\/(.*) differ).freeze + class << self def between(repo, head, base, options = {}, *paths) straight = options.delete(:straight) || false @@ -131,8 +133,13 @@ module Gitlab def patch_hard_limit_bytes Gitlab::CurrentSettings.diff_max_patch_bytes end - end + def has_binary_notice?(text) + return false unless text.present? + + text.start_with?(BINARY_NOTICE_PATTERN) + end + end def initialize(raw_diff, expanded: true) @expanded = expanded @@ -215,7 +222,7 @@ module Gitlab end def has_binary_notice? - @diff.start_with?('Binary') + self.class.has_binary_notice?(@diff) end private diff --git a/lib/gitlab/http.rb b/lib/gitlab/http.rb index be87dcc0ff9..7e45cd216f5 100644 --- a/lib/gitlab/http.rb +++ b/lib/gitlab/http.rb @@ -8,9 +8,10 @@ module Gitlab class HTTP BlockedUrlError = Class.new(StandardError) RedirectionTooDeep = Class.new(StandardError) + ReadTotalTimeout = Class.new(Net::ReadTimeout) HTTP_TIMEOUT_ERRORS = [ - Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout + Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Gitlab::HTTP::ReadTotalTimeout ].freeze HTTP_ERRORS = HTTP_TIMEOUT_ERRORS + [ SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, @@ -23,6 +24,7 @@ module Gitlab read_timeout: 20, write_timeout: 30 }.freeze + DEFAULT_READ_TOTAL_TIMEOUT = 20.seconds include HTTParty # rubocop:disable Gitlab/HTTParty @@ -41,7 +43,19 @@ module Gitlab options end - httparty_perform_request(http_method, path, options_with_timeouts, &block) + unless options.has_key?(:use_read_total_timeout) + return httparty_perform_request(http_method, path, options_with_timeouts, &block) + end + + start_time = Gitlab::Metrics::System.monotonic_time + read_total_timeout = options.fetch(:timeout, DEFAULT_READ_TOTAL_TIMEOUT) + + httparty_perform_request(http_method, path, options_with_timeouts) do |fragment| + elapsed = Gitlab::Metrics::System.monotonic_time - start_time + raise ReadTotalTimeout, "Request timed out after #{elapsed} seconds" if elapsed > read_total_timeout + + block.call fragment if block + end rescue HTTParty::RedirectionTooDeep raise RedirectionTooDeep rescue *HTTP_ERRORS => e diff --git a/locale/gitlab.pot b/locale/gitlab.pot index f34bea613e8..b82dc3d5259 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -14032,6 +14032,9 @@ msgstr "" msgid "File renamed with no changes." msgstr "" +msgid "File suppressed by a .gitattributes entry or the file's encoding is unsupported." +msgstr "" + msgid "File synchronization concurrency limit" msgstr "" @@ -33273,9 +33276,6 @@ msgstr "" msgid "This diff is collapsed." msgstr "" -msgid "This diff was suppressed by a .gitattributes entry." -msgstr "" - msgid "This directory" msgstr "" diff --git a/spec/features/expand_collapse_diffs_spec.rb b/spec/features/expand_collapse_diffs_spec.rb index cbd1ae628d1..add4af2bcdb 100644 --- a/spec/features/expand_collapse_diffs_spec.rb +++ b/spec/features/expand_collapse_diffs_spec.rb @@ -253,7 +253,7 @@ RSpec.describe 'Expand and collapse diffs', :js do click_link('Expand all') # Wait for elements to appear to ensure full page reload - expect(page).to have_content('This diff was suppressed by a .gitattributes entry') + expect(page).to have_content("File suppressed by a .gitattributes entry or the file's encoding is unsupported.") expect(page).to have_content('This source diff could not be displayed because it is too large.') expect(page).to have_content('too_large_image.jpg') find('.note-textarea') diff --git a/spec/features/projects/diffs/diff_show_spec.rb b/spec/features/projects/diffs/diff_show_spec.rb index e47f36c4b7a..56506ada3ce 100644 --- a/spec/features/projects/diffs/diff_show_spec.rb +++ b/spec/features/projects/diffs/diff_show_spec.rb @@ -174,4 +174,14 @@ RSpec.describe 'Diff file viewer', :js, :with_clean_rails_cache do end end end + + context 'when the the encoding of the file is unsupported' do + before do + visit_commit('f05a98786e4274708e1fa118c7ad3a29d1d1b9a3') + end + + it 'shows it is not diffable' do + expect(page).to have_content("File suppressed by a .gitattributes entry or the file's encoding is unsupported.") + end + end end diff --git a/spec/lib/gitlab/diff/file_collection/merge_request_diff_spec.rb b/spec/lib/gitlab/diff/file_collection/merge_request_diff_spec.rb index 03a9b9bd21e..d401c42fed7 100644 --- a/spec/lib/gitlab/diff/file_collection/merge_request_diff_spec.rb +++ b/spec/lib/gitlab/diff/file_collection/merge_request_diff_spec.rb @@ -23,7 +23,7 @@ RSpec.describe Gitlab::Diff::FileCollection::MergeRequestDiff do it 'does not highlight files marked as undiffable in .gitattributes' do allow_next_instance_of(Gitlab::Diff::File) do |instance| - allow(instance).to receive(:diffable?).and_return(false) + allow(instance).to receive(:diffable_by_attribute?).and_return(false) end expect_next_instance_of(Gitlab::Diff::File) do |instance| diff --git a/spec/lib/gitlab/diff/file_spec.rb b/spec/lib/gitlab/diff/file_spec.rb index 78be89c449b..1800d2d6b60 100644 --- a/spec/lib/gitlab/diff/file_spec.rb +++ b/spec/lib/gitlab/diff/file_spec.rb @@ -186,26 +186,46 @@ RSpec.describe Gitlab::Diff::File do end describe '#diffable?' do - let(:commit) { project.commit('1a0b36b3cdad1d2ee32457c102a8c0b7056fa863') } - let(:diffs) { commit.diffs } + context 'when attributes exist' do + let(:commit) { project.commit('1a0b36b3cdad1d2ee32457c102a8c0b7056fa863') } + let(:diffs) { commit.diffs } - before do - info_dir_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do - File.join(project.repository.path_to_repo, 'info') + before do + info_dir_path = Gitlab::GitalyClient::StorageSettings.allow_disk_access do + File.join(project.repository.path_to_repo, 'info') + end + + FileUtils.mkdir(info_dir_path) unless File.exist?(info_dir_path) + File.write(File.join(info_dir_path, 'attributes'), "*.md -diff\n") end - FileUtils.mkdir(info_dir_path) unless File.exist?(info_dir_path) - File.write(File.join(info_dir_path, 'attributes'), "*.md -diff\n") + it "returns true for files that do not have attributes" do + diff_file = diffs.diff_file_with_new_path('LICENSE') + expect(diff_file.diffable?).to be_truthy + end + + it "returns false for files that have been marked as not being diffable in attributes" do + diff_file = diffs.diff_file_with_new_path('README.md') + expect(diff_file.diffable?).to be_falsey + end end - it "returns true for files that do not have attributes" do - diff_file = diffs.diff_file_with_new_path('LICENSE') - expect(diff_file.diffable?).to be_truthy + context 'when the text has binary notice' do + let(:commit) { project.commit('f05a98786e4274708e1fa118c7ad3a29d1d1b9a3') } + let(:diff_file) { commit.diffs.diff_file_with_new_path('VERSION') } + + it "returns false" do + expect(diff_file.diffable?).to be_falsey + end end - it "returns false for files that have been marked as not being diffable in attributes" do - diff_file = diffs.diff_file_with_new_path('README.md') - expect(diff_file.diffable?).to be_falsey + context 'when the content is binary' do + let(:commit) { project.commit('2f63565e7aac07bcdadb654e253078b727143ec4') } + let(:diff_file) { commit.diffs.diff_file_with_new_path('files/images/6049019_460s.jpg') } + + it "returns true" do + expect(diff_file.diffable?).to be_truthy + end end end @@ -729,6 +749,18 @@ RSpec.describe Gitlab::Diff::File do end end + context 'when the the encoding of the file is unsupported' do + let(:commit) { project.commit('f05a98786e4274708e1fa118c7ad3a29d1d1b9a3') } + let(:diff_file) { commit.diffs.diff_file_with_new_path('VERSION') } + + it 'returns a Not Diffable viewer' do + expect(diff_file.simple_viewer).to be_a(DiffViewer::NotDiffable) + end + + it { expect(diff_file.highlighted_diff_lines).to eq([]) } + it { expect(diff_file.parallel_diff_lines).to eq([]) } + end + describe '#diff_hunk' do context 'when first line is a match' do let(:raw_diff) do diff --git a/spec/lib/gitlab/diff/parser_spec.rb b/spec/lib/gitlab/diff/parser_spec.rb index 7448ae0b2ea..c8069f82f04 100644 --- a/spec/lib/gitlab/diff/parser_spec.rb +++ b/spec/lib/gitlab/diff/parser_spec.rb @@ -146,6 +146,16 @@ eos it { expect(parser.parse(nil)).to eq([]) } end + context 'when it is a binary notice' do + let(:diff) do + <<~END + Binary files a/test and b/test differ + END + end + + it { expect(parser.parse(diff.each_line)).to eq([]) } + end + describe 'tolerates special diff markers in a content' do it "counts lines correctly" do diff = <<~END diff --git a/spec/lib/gitlab/http_spec.rb b/spec/lib/gitlab/http_spec.rb index 308f7f46251..71e80de9f89 100644 --- a/spec/lib/gitlab/http_spec.rb +++ b/spec/lib/gitlab/http_spec.rb @@ -27,6 +27,47 @@ RSpec.describe Gitlab::HTTP do end end + context 'when reading the response is too slow' do + before do + stub_const("#{described_class}::DEFAULT_READ_TOTAL_TIMEOUT", 0.001.seconds) + + WebMock.stub_request(:post, /.*/).to_return do |request| + sleep 0.002.seconds + { body: 'I\m slow', status: 200 } + end + end + + let(:options) { {} } + + subject(:request_slow_responder) { described_class.post('http://example.org', **options) } + + specify do + expect { request_slow_responder }.not_to raise_error + end + + context 'with use_read_total_timeout option' do + let(:options) { { use_read_total_timeout: true } } + + it 'raises a timeout error' do + expect { request_slow_responder }.to raise_error(Gitlab::HTTP::ReadTotalTimeout, /Request timed out after ?([0-9]*[.])?[0-9]+ seconds/) + end + + context 'and timeout option' do + let(:options) { { use_read_total_timeout: true, timeout: 10.seconds } } + + it 'overrides the default timeout when timeout option is present' do + expect { request_slow_responder }.not_to raise_error + end + end + end + end + + it 'calls a block' do + WebMock.stub_request(:post, /.*/) + + expect { |b| described_class.post('http://example.org', &b) }.to yield_with_args + end + describe 'allow_local_requests_from_web_hooks_and_services is' do before do WebMock.stub_request(:get, /.*/).to_return(status: 200, body: 'Success') diff --git a/spec/models/protected_branch/push_access_level_spec.rb b/spec/models/protected_branch/push_access_level_spec.rb index 17a589f0485..fa84cd660cb 100644 --- a/spec/models/protected_branch/push_access_level_spec.rb +++ b/spec/models/protected_branch/push_access_level_spec.rb @@ -44,7 +44,7 @@ RSpec.describe ProtectedBranch::PushAccessLevel do let(:can_push) { true } before_all do - project.add_guest(user) + project.add_maintainer(user) end context 'when this push_access_level is tied to a deploy key' do -- cgit v1.2.3