diff options
Diffstat (limited to 'gems/gitlab-http/spec/gitlab')
11 files changed, 2209 insertions, 0 deletions
diff --git a/gems/gitlab-http/spec/gitlab/http_v2/buffered_io_spec.rb b/gems/gitlab-http/spec/gitlab/http_v2/buffered_io_spec.rb new file mode 100644 index 00000000000..74fe8b18199 --- /dev/null +++ b/gems/gitlab-http/spec/gitlab/http_v2/buffered_io_spec.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::HTTP_V2::BufferedIo do + describe '#readuntil' do + let(:mock_io) { StringIO.new('a') } + let(:start_time) { Process.clock_gettime(Process::CLOCK_MONOTONIC) } + + before do + stub_const('Gitlab::HTTP_V2::BufferedIo::HEADER_READ_TIMEOUT', 0.1) + end + + subject(:readuntil) do + described_class.new(mock_io).readuntil('a', false, start_time) + end + + it 'does not raise a timeout error' do + expect { readuntil }.not_to raise_error + end + + context 'when the response contains infinitely long headers' do + before do + read_counter = 0 + + allow(mock_io).to receive(:read_nonblock) do |buffer_size, *_| + read_counter += 1 + raise 'Test did not raise HeaderReadTimeout' if read_counter > 10 + + sleep 0.01 + 'H' * buffer_size + end + end + + it 'raises a timeout error' do + expect do + readuntil + end.to raise_error(Gitlab::HTTP_V2::HeaderReadTimeout, + /Request timed out after reading headers for 0\.[0-9]+ seconds/) + end + + context 'when not passing start_time' do + subject(:readuntil) do + described_class.new(mock_io).readuntil('a', false) + end + + it 'raises a timeout error' do + expect do + readuntil + end.to raise_error(Gitlab::HTTP_V2::HeaderReadTimeout, + /Request timed out after reading headers for 0\.[0-9]+ seconds/) + end + end + end + end +end diff --git a/gems/gitlab-http/spec/gitlab/http_v2/domain_allowlist_entry_spec.rb b/gems/gitlab-http/spec/gitlab/http_v2/domain_allowlist_entry_spec.rb new file mode 100644 index 00000000000..0f9d5bc550d --- /dev/null +++ b/gems/gitlab-http/spec/gitlab/http_v2/domain_allowlist_entry_spec.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::HTTP_V2::DomainAllowlistEntry do + let(:domain) { 'www.example.com' } + + describe '#initialize' do + it 'initializes without port' do + domain_allowlist_entry = described_class.new(domain) + + expect(domain_allowlist_entry.domain).to eq(domain) + expect(domain_allowlist_entry.port).to be(nil) + end + + it 'initializes with port' do + port = 8080 + domain_allowlist_entry = described_class.new(domain, port: port) + + expect(domain_allowlist_entry.domain).to eq(domain) + expect(domain_allowlist_entry.port).to eq(port) + end + end + + describe '#match?' do + it 'matches when domain and port are equal' do + port = 8080 + domain_allowlist_entry = described_class.new(domain, port: port) + + expect(domain_allowlist_entry).to be_match(domain, port) + end + + it 'matches any port when port is nil' do + domain_allowlist_entry = described_class.new(domain) + + expect(domain_allowlist_entry).to be_match(domain, 8080) + expect(domain_allowlist_entry).to be_match(domain, 9090) + end + + it 'does not match when port is present but requested_port is nil' do + domain_allowlist_entry = described_class.new(domain, port: 8080) + + expect(domain_allowlist_entry).not_to be_match(domain, nil) + end + + it 'matches when port and requested_port are nil' do + domain_allowlist_entry = described_class.new(domain) + + expect(domain_allowlist_entry).to be_match(domain) + end + + it 'does not match if domain is not equal' do + domain_allowlist_entry = described_class.new(domain) + + expect(domain_allowlist_entry).not_to be_match('www.gitlab.com', 8080) + end + end +end diff --git a/gems/gitlab-http/spec/gitlab/http_v2/ip_allowlist_entry_spec.rb b/gems/gitlab-http/spec/gitlab/http_v2/ip_allowlist_entry_spec.rb new file mode 100644 index 00000000000..ad7d993ec62 --- /dev/null +++ b/gems/gitlab-http/spec/gitlab/http_v2/ip_allowlist_entry_spec.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::HTTP_V2::IpAllowlistEntry, feature_category: :shared do + let(:ipv4) { IPAddr.new('192.168.1.1') } + + describe '#initialize' do + it 'initializes without port' do + ip_allowlist_entry = described_class.new(ipv4) + + expect(ip_allowlist_entry.ip).to eq(ipv4) + expect(ip_allowlist_entry.port).to be(nil) + end + + it 'initializes with port' do + port = 8080 + ip_allowlist_entry = described_class.new(ipv4, port: port) + + expect(ip_allowlist_entry.ip).to eq(ipv4) + expect(ip_allowlist_entry.port).to eq(port) + end + end + + describe '#match?' do + it 'matches with equivalent IP and port' do + port = 8080 + ip_allowlist_entry = described_class.new(ipv4, port: port) + + expect(ip_allowlist_entry).to be_match(ipv4.to_s, port) + end + + it 'matches any port when port is nil' do + ip_allowlist_entry = described_class.new(ipv4) + + expect(ip_allowlist_entry).to be_match(ipv4.to_s, 8080) + expect(ip_allowlist_entry).to be_match(ipv4.to_s, 9090) + end + + it 'does not match when port is present but requested_port is nil' do + ip_allowlist_entry = described_class.new(ipv4, port: 8080) + + expect(ip_allowlist_entry).not_to be_match(ipv4.to_s, nil) + end + + it 'matches when port and requested_port are nil' do + ip_allowlist_entry = described_class.new(ipv4) + + expect(ip_allowlist_entry).to be_match(ipv4.to_s) + end + + it 'works with ipv6' do + ipv6 = IPAddr.new('fe80::c800:eff:fe74:8') + ip_allowlist_entry = described_class.new(ipv6) + + expect(ip_allowlist_entry).to be_match(ipv6.to_s, 8080) + end + + it 'matches ipv4 within IPv4 range' do + ipv4_range = IPAddr.new('127.0.0.0/28') + ip_allowlist_entry = described_class.new(ipv4_range) + + expect(ip_allowlist_entry).to be_match(ipv4_range.to_range.last.to_s, 8080) + expect(ip_allowlist_entry).not_to be_match('127.0.1.1', 8080) + end + + it 'matches IPv6 within IPv6 range' do + ipv6_range = IPAddr.new('::ffff:192.168.1.0/8') + ip_allowlist_entry = described_class.new(ipv6_range) + + expect(ip_allowlist_entry).to be_match(ipv6_range.to_range.last.to_s, 8080) + expect(ip_allowlist_entry).not_to be_match('fd84:6d02:f6d8:f::f', 8080) + end + + it 'matches IPv4 to IPv6 mapped addresses in allow list' do + ipv6_range = IPAddr.new('::ffff:192.168.1.1') + ip_allowlist_entry = described_class.new(ipv6_range) + + expect(ip_allowlist_entry).to be_match(ipv4, 8080) + expect(ip_allowlist_entry).to be_match(ipv6_range.to_range.last.to_s, 8080) + expect(ip_allowlist_entry).not_to be_match('::ffff:192.168.1.0', 8080) + expect(ip_allowlist_entry).not_to be_match('::ffff:169.254.168.101', 8080) + end + + it 'matches IPv4 to IPv6 mapped addresses in requested IP' do + ipv4_range = IPAddr.new('192.168.1.1/24') + ip_allowlist_entry = described_class.new(ipv4_range) + + expect(ip_allowlist_entry).to be_match(ipv4, 8080) + expect(ip_allowlist_entry).to be_match('::ffff:192.168.1.0', 8080) + expect(ip_allowlist_entry).to be_match('::ffff:192.168.1.1', 8080) + expect(ip_allowlist_entry).not_to be_match('::ffff:169.254.170.100/8', 8080) + end + end +end diff --git a/gems/gitlab-http/spec/gitlab/http_v2/net_http_adapter_spec.rb b/gems/gitlab-http/spec/gitlab/http_v2/net_http_adapter_spec.rb new file mode 100644 index 00000000000..22998803cc8 --- /dev/null +++ b/gems/gitlab-http/spec/gitlab/http_v2/net_http_adapter_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'net/http' + +RSpec.describe Gitlab::HTTP_V2::NetHttpAdapter, feature_category: :api do + describe '#connect' do + let(:url) { 'https://example.org' } + let(:net_http_adapter) { described_class.new(url) } + + subject(:connect) { net_http_adapter.send(:connect) } + + before do + allow(TCPSocket).to receive(:open).and_return(Socket.new(:INET, :STREAM)) + end + + it 'uses a Gitlab::HTTP_V2::BufferedIo instance as @socket' do + connect + + expect(net_http_adapter.instance_variable_get(:@socket)).to be_a(Gitlab::HTTP_V2::BufferedIo) + end + end +end diff --git a/gems/gitlab-http/spec/gitlab/http_v2/net_http_patch_spec.rb b/gems/gitlab-http/spec/gitlab/http_v2/net_http_patch_spec.rb new file mode 100644 index 00000000000..b82646fb365 --- /dev/null +++ b/gems/gitlab-http/spec/gitlab/http_v2/net_http_patch_spec.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +require 'spec_helper' +require 'net/http' + +RSpec.describe 'Net::HTTP patch proxy user and password encoding' do + let(:net_http) { Net::HTTP.new('hostname.example') } + + before do + # This file can be removed once Ruby 3.0 is no longer supported: + # https://gitlab.com/gitlab-org/gitlab/-/issues/396223 + skip if Gem::Version.new(Net::HTTP::VERSION) >= Gem::Version.new('0.2.0') + end + + describe '#proxy_user' do + subject { net_http.proxy_user } + + it { is_expected.to eq(nil) } + + context 'with http_proxy env' do + let(:http_proxy) { 'http://proxy.example:8000' } + + before do + stub_env('http_proxy', http_proxy) + end + + it { is_expected.to eq(nil) } + + context 'and user:password authentication' do + let(:http_proxy) { 'http://Y%5CX:R%25S%5D%20%3FX@proxy.example:8000' } + + context 'when on multiuser safe platform' do + # linux, freebsd, darwin are considered multi user safe platforms + # See https://github.com/ruby/net-http/blob/v0.1.1/lib/net/http.rb#L1174-L1178 + + before do + allow(net_http).to receive(:environment_variable_is_multiuser_safe?).and_return(true) + end + + it { is_expected.to eq 'Y\\X' } + end + + context 'when not on multiuser safe platform' do + before do + allow(net_http).to receive(:environment_variable_is_multiuser_safe?).and_return(false) + end + + it { is_expected.to be_nil } + end + end + end + end + + describe '#proxy_pass' do + subject { net_http.proxy_pass } + + it { is_expected.to eq(nil) } + + context 'with http_proxy env' do + let(:http_proxy) { 'http://proxy.example:8000' } + + before do + stub_env('http_proxy', http_proxy) + end + + it { is_expected.to eq(nil) } + + context 'and user:password authentication' do + let(:http_proxy) { 'http://Y%5CX:R%25S%5D%20%3FX@proxy.example:8000' } + + context 'when on multiuser safe platform' do + # linux, freebsd, darwin are considered multi user safe platforms + # See https://github.com/ruby/net-http/blob/v0.1.1/lib/net/http.rb#L1174-L1178 + + before do + allow(net_http).to receive(:environment_variable_is_multiuser_safe?).and_return(true) + end + + it { is_expected.to eq 'R%S] ?X' } + end + + context 'when not on multiuser safe platform' do + before do + allow(net_http).to receive(:environment_variable_is_multiuser_safe?).and_return(false) + end + + it { is_expected.to be_nil } + end + end + end + end +end diff --git a/gems/gitlab-http/spec/gitlab/http_v2/net_http_response_patch_spec.rb b/gems/gitlab-http/spec/gitlab/http_v2/net_http_response_patch_spec.rb new file mode 100644 index 00000000000..f8d0f0a57fc --- /dev/null +++ b/gems/gitlab-http/spec/gitlab/http_v2/net_http_response_patch_spec.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'Net::HTTPResponse patch header read timeout', feature_category: :shared do + describe '.each_response_header' do + let(:server_response) do + <<~HTTP + Content-Type: text/html + Header-Two: foo + + Hello World + HTTP + end + + before do + stub_const('Gitlab::HTTP_V2::BufferedIo::HEADER_READ_TIMEOUT', 0.1) + end + + subject(:each_response_header) { Net::HTTPResponse.each_response_header(socket) { |k, v| } } # rubocop:disable Lint/EmptyBlock + + context 'with Net::BufferedIO' do + let(:socket) { Net::BufferedIO.new(StringIO.new(server_response)) } + + it 'does not forward start time to the socket' do + allow(socket).to receive(:readuntil).and_call_original + expect(socket).to receive(:readuntil).with("\n", true) + + each_response_header + end + + context 'when the response contains many consecutive spaces' do + it 'has no regex backtracking issues' do + expect(socket).to receive(:readuntil).and_return( + "a: #{' ' * 100_000} b", + '' + ) + + Timeout.timeout(1) do + each_response_header + end + end + end + end + + context 'with Gitlab:HTTP_V2:::BufferedIo' do + let(:mock_io) { StringIO.new(server_response) } + let(:socket) { Gitlab::HTTP_V2::BufferedIo.new(mock_io) } + + it 'forwards start time to the socket' do + allow(socket).to receive(:readuntil).and_call_original + expect(socket).to receive(:readuntil).with("\n", true, kind_of(Numeric)) + + each_response_header + end + + context 'when the response contains an infinite number of headers' do + before do + read_counter = 0 + + allow(mock_io).to receive(:read_nonblock) do + read_counter += 1 + raise 'Test did not raise HeaderReadTimeout' if read_counter > 10 + + sleep 0.01 + +"Yet-Another-Header: foo\n" + end + end + + it 'raises a timeout error' do + expect { each_response_header }.to raise_error(Gitlab::HTTP_V2::HeaderReadTimeout, + /Request timed out after reading headers for 0\.[0-9]+ seconds/) + end + end + end + end +end diff --git a/gems/gitlab-http/spec/gitlab/http_v2/new_connection_adapter_spec.rb b/gems/gitlab-http/spec/gitlab/http_v2/new_connection_adapter_spec.rb new file mode 100644 index 00000000000..852bafc5557 --- /dev/null +++ b/gems/gitlab-http/spec/gitlab/http_v2/new_connection_adapter_spec.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::HTTP_V2::NewConnectionAdapter, feature_category: :shared do + let(:uri) { URI('https://example.org') } + let(:options) { {} } + + subject(:connection) { described_class.new(uri, options).connection } + + describe '#connection' do + before do + stub_all_dns('https://example.org', ip_address: '93.184.216.34') + end + + context 'when local requests are allowed' do + let(:options) { { allow_local_requests: true } } + + it 'sets up the connection' do + expect(connection).to be_a(Gitlab::HTTP_V2::NetHttpAdapter) + expect(connection.address).to eq('93.184.216.34') + expect(connection.hostname_override).to eq('example.org') + expect(connection.addr_port).to eq('example.org') + expect(connection.port).to eq(443) + end + end + + context 'when local requests are not allowed' do + let(:options) { { allow_local_requests: false } } + + it 'sets up the connection' do + expect(connection).to be_a(Gitlab::HTTP_V2::NetHttpAdapter) + expect(connection.address).to eq('93.184.216.34') + expect(connection.hostname_override).to eq('example.org') + expect(connection.addr_port).to eq('example.org') + expect(connection.port).to eq(443) + end + + context 'when it is a request to local network' do + let(:uri) { URI('http://172.16.0.0/12') } + + it 'raises error' do + expect { subject }.to raise_error( + Gitlab::HTTP_V2::BlockedUrlError, + "URL is blocked: Requests to the local network are not allowed" + ) + end + + context 'when local request allowed' do + let(:options) { { allow_local_requests: true } } + + it 'sets up the connection' do + expect(connection).to be_a(Gitlab::HTTP_V2::NetHttpAdapter) + expect(connection.address).to eq('172.16.0.0') + expect(connection.hostname_override).to be(nil) + expect(connection.addr_port).to eq('172.16.0.0') + expect(connection.port).to eq(80) + end + end + end + + context 'when it is a request to local address' do + let(:uri) { URI('http://127.0.0.1') } + + it 'raises error' do + expect { subject }.to raise_error( + Gitlab::HTTP_V2::BlockedUrlError, + "URL is blocked: Requests to localhost are not allowed" + ) + end + + context 'when local request allowed' do + let(:options) { { allow_local_requests: true } } + + it 'sets up the connection' do + expect(connection).to be_a(Gitlab::HTTP_V2::NetHttpAdapter) + expect(connection.address).to eq('127.0.0.1') + expect(connection.hostname_override).to be(nil) + expect(connection.addr_port).to eq('127.0.0.1') + expect(connection.port).to eq(80) + end + end + end + + context 'when port different from URL scheme is used' do + let(:uri) { URI('https://example.org:8080') } + + it 'sets up the addr_port accordingly' do + expect(connection).to be_a(Gitlab::HTTP_V2::NetHttpAdapter) + expect(connection.address).to eq('93.184.216.34') + expect(connection.hostname_override).to eq('example.org') + expect(connection.addr_port).to eq('example.org:8080') + expect(connection.port).to eq(8080) + end + end + end + + context 'when DNS rebinding protection is disabled' do + let(:options) { { dns_rebinding_protection_enabled: false } } + + it 'sets up the connection' do + expect(connection).to be_a(Gitlab::HTTP_V2::NetHttpAdapter) + expect(connection.address).to eq('example.org') + expect(connection.hostname_override).to eq(nil) + expect(connection.addr_port).to eq('example.org') + expect(connection.port).to eq(443) + end + end + + context 'when proxy is enabled' do + before do + stub_env('http_proxy', 'http://proxy.example.com') + end + + it 'proxy stays configured' do + expect(connection.proxy?).to be true + expect(connection.proxy_from_env?).to be true + expect(connection.proxy_address).to eq('proxy.example.com') + end + + context 'when no_proxy matches the request' do + before do + stub_env('no_proxy', 'example.org') + end + + it 'proxy is disabled' do + expect(connection.proxy?).to be false + expect(connection.proxy_from_env?).to be false + expect(connection.proxy_address).to be nil + end + end + + context 'when no_proxy does not match the request' do + before do + stub_env('no_proxy', 'example.com') + end + + it 'proxy stays configured' do + expect(connection.proxy?).to be true + expect(connection.proxy_from_env?).to be true + expect(connection.proxy_address).to eq('proxy.example.com') + end + end + end + + context 'when URL scheme is not HTTP/HTTPS' do + let(:uri) { URI('ssh://example.org') } + + it 'raises error' do + expect { subject }.to raise_error( + Gitlab::HTTP_V2::BlockedUrlError, + "URL is blocked: Only allowed schemes are http, https" + ) + end + end + end +end diff --git a/gems/gitlab-http/spec/gitlab/http_v2/url_allowlist_spec.rb b/gems/gitlab-http/spec/gitlab/http_v2/url_allowlist_spec.rb new file mode 100644 index 00000000000..bac69a2c38c --- /dev/null +++ b/gems/gitlab-http/spec/gitlab/http_v2/url_allowlist_spec.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::HTTP_V2::UrlAllowlist do + let(:allowlist) { [] } + + describe '#domain_allowed?' do + let(:allowlist) { %w[www.example.com example.com] } + + it 'returns true if domains present in allowlist' do + not_allowed = %w[subdomain.example.com example.org] + + aggregate_failures do + allowlist.each do |domain| + expect(described_class).to be_domain_allowed(domain, allowlist) + end + + not_allowed.each do |domain| + expect(described_class).not_to be_domain_allowed(domain, allowlist) + end + end + end + + it 'returns false when domain is blank' do + expect(described_class).not_to be_domain_allowed(nil, allowlist) + end + + context 'with ports' do + let(:allowlist) { ['example.io:3000'] } + + it 'returns true if domain and ports present in allowlist' do + parsed_allowlist = [['example.io', 3000]] + not_allowed = [ + 'example.io', + ['example.io', 3001] + ] + + aggregate_failures do + parsed_allowlist.each do |domain, port| + expect(described_class).to be_domain_allowed(domain, allowlist, port: port) + end + + not_allowed.each do |domain, port| + expect(described_class).not_to be_domain_allowed(domain, allowlist, port: port) + end + end + end + end + end + + describe '#ip_allowed?' do + let(:allowlist) do + [ + '0.0.0.0', + '127.0.0.1', + '192.168.1.1', + '0:0:0:0:0:ffff:192.168.1.2', + '::ffff:c0a8:102', + 'fc00:bf8b:e62c:abcd:abcd:aaaa:aaaa:aaaa', + '0:0:0:0:0:ffff:169.254.169.254', + '::ffff:a9fe:a9fe', + '::ffff:a9fe:a864', + 'fe80::c800:eff:fe74:8' + ] + end + + it 'returns true if ips present in allowlist' do + aggregate_failures do + allowlist.each do |ip_address| + expect(described_class).to be_ip_allowed(ip_address, allowlist) + end + + %w[172.16.2.2 127.0.0.2 fe80::c800:eff:fe74:9].each do |ip_address| + expect(described_class).not_to be_ip_allowed(ip_address, allowlist) + end + end + end + + it 'returns false when ip is blank' do + expect(described_class).not_to be_ip_allowed(nil, allowlist) + end + + context 'with ip ranges in allowlist' do + let(:ipv4_range) { '127.0.0.0/28' } + let(:ipv6_range) { 'fd84:6d02:f6d8:c89e::/124' } + + let(:allowlist) do + [ + ipv4_range, + ipv6_range + ] + end + + it 'does not allowlist ipv4 range when not in allowlist' do + IPAddr.new(ipv4_range).to_range.to_a.each do |ip| + expect(described_class).not_to be_ip_allowed(ip.to_s, []) + end + end + + it 'allowlists all ipv4s in the range when in allowlist' do + IPAddr.new(ipv4_range).to_range.to_a.each do |ip| + expect(described_class).to be_ip_allowed(ip.to_s, allowlist) + end + end + + it 'does not allowlist ipv6 range when not in allowlist' do + IPAddr.new(ipv6_range).to_range.to_a.each do |ip| + expect(described_class).not_to be_ip_allowed(ip.to_s, []) + end + end + + it 'allowlists all ipv6s in the range when in allowlist' do + IPAddr.new(ipv6_range).to_range.to_a.each do |ip| + expect(described_class).to be_ip_allowed(ip.to_s, allowlist) + end + end + + it 'does not allowlist IPs outside the range' do + expect(described_class).not_to be_ip_allowed("fd84:6d02:f6d8:c89e:0:0:1:f", allowlist) + + expect(described_class).not_to be_ip_allowed("127.0.1.15", allowlist) + end + end + + context 'with ports' do + let(:allowlist) { %w[127.0.0.9:3000 [2001:db8:85a3:8d3:1319:8a2e:370:7348]:443] } + + it 'returns true if ip and ports present in allowlist' do + parsed_allowlist = [ + ['127.0.0.9', 3000], + ['[2001:db8:85a3:8d3:1319:8a2e:370:7348]', 443] + ] + not_allowed = [ + '127.0.0.9', + ['127.0.0.9', 3001], + '[2001:db8:85a3:8d3:1319:8a2e:370:7348]', + ['[2001:db8:85a3:8d3:1319:8a2e:370:7348]', 3001] + ] + + aggregate_failures do + parsed_allowlist.each do |ip, port| + expect(described_class).to be_ip_allowed(ip, allowlist, port: port) + end + + not_allowed.each do |ip, port| + expect(described_class).not_to be_ip_allowed(ip, allowlist, port: port) + end + end + end + end + end +end diff --git a/gems/gitlab-http/spec/gitlab/http_v2/url_blocker_spec.rb b/gems/gitlab-http/spec/gitlab/http_v2/url_blocker_spec.rb new file mode 100644 index 00000000000..e47098e6f74 --- /dev/null +++ b/gems/gitlab-http/spec/gitlab/http_v2/url_blocker_spec.rb @@ -0,0 +1,988 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::HTTP_V2::UrlBlocker, :stub_invalid_dns_only, feature_category: :shared do + let(:schemes) { %w[http https] } + + # This test ensures backward compatibliity for the validate! method. + # We shoud refactor all callers of validate! to handle a Result object: + # https://gitlab.com/gitlab-org/gitlab/-/issues/410890 + describe '#validate!' do + let(:options) { { schemes: schemes } } + + subject { described_class.validate!(import_url, **options) } + + shared_examples 'validates URI and hostname' do + it 'runs the url validations' do + uri, hostname = subject + + expect(uri).to eq(Addressable::URI.parse(expected_uri)) + expect(hostname).to eq(expected_hostname) + end + end + + context 'when the URL hostname is a domain' do + context 'when domain can be resolved' do + let(:import_url) { 'https://example.org' } + + before do + stub_dns(import_url, ip_address: '93.184.216.34') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'https://93.184.216.34' } + let(:expected_hostname) { 'example.org' } + let(:expected_use_proxy) { false } + end + end + end + end + + describe '#validate_url_with_proxy!' do + let(:options) { { schemes: schemes } } + + subject { described_class.validate_url_with_proxy!(import_url, **options) } + + shared_examples 'validates URI and hostname' do + it 'runs the url validations' do + expect(subject.uri).to eq(Addressable::URI.parse(expected_uri)) + expect(subject.hostname).to eq(expected_hostname) + expect(subject.use_proxy).to eq(expected_use_proxy) + end + end + + shared_context 'when instance configured to deny all requests' do + let(:options) { super().merge(deny_all_requests_except_allowed: true) } + end + + shared_examples 'a URI denied by `deny_all_requests_except_allowed`' do + context 'when instance setting is enabled' do + include_context 'when instance configured to deny all requests' + + it 'blocks the request' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + + context 'when instance setting is not enabled' do + it 'does not block the request' do + expect { subject }.not_to raise_error + end + end + + context 'when passed as an argument' do + let(:options) { super().merge(deny_all_requests_except_allowed: arg_value) } + + context 'when argument is a proc that evaluates to true' do + let(:arg_value) { proc { true } } + + it 'blocks the request' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + + context 'when argument is a proc that evaluates to false' do + let(:arg_value) { proc { false } } + + it 'does not block the request' do + expect { subject }.not_to raise_error + end + end + + context 'when argument is true' do + let(:arg_value) { true } + + it 'blocks the request' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + + context 'when argument is false' do + let(:arg_value) { false } + + it 'does not block the request' do + expect { subject }.not_to raise_error + end + end + end + end + + shared_examples 'a URI exempt from `deny_all_requests_except_allowed`' do + include_context 'when instance configured to deny all requests' + + it 'does not block the request' do + expect { subject }.not_to raise_error + end + end + + context 'when URI is nil' do + let(:import_url) { nil } + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { nil } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { true } + end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' + end + + context 'when URI is internal' do + let(:import_url) { 'http://localhost' } + + before do + stub_dns(import_url, ip_address: '127.0.0.1') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'http://127.0.0.1' } + let(:expected_hostname) { 'localhost' } + let(:expected_use_proxy) { false } + end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' + end + + context 'when URI is for a local object storage' do + let(:import_url) { "#{host}/external-diffs/merge_request_diffs/mr-1/diff-1" } + + context 'when extra_allowed_uris is passed' do + let(:options) { super().merge(extra_allowed_uris: [URI(host)]) } + + context 'with a local domain name' do + let(:host) { 'http://review-minio-svc.svc:9000' } + + before do + stub_dns(host, ip_address: '127.0.0.1') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'http://127.0.0.1:9000/external-diffs/merge_request_diffs/mr-1/diff-1' } + let(:expected_hostname) { 'review-minio-svc.svc' } + let(:expected_use_proxy) { false } + end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' + end + + context 'with an IP address' do + let(:host) { 'http://127.0.0.1:9000' } + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'http://127.0.0.1:9000/external-diffs/merge_request_diffs/mr-1/diff-1' } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } + end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' + end + + context 'with an LFS object storage' do + let(:host) { 'http://127.0.0.1:9000' } + + context 'when extra_allowed_uris is not passed' do + let(:options) { super().merge(extra_allowed_uris: []) } + + it 'raises an error' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + end + end + + context 'when extra_allowed_uris is not passed' do + context 'with a local domain name' do + let(:host) { 'http://review-minio-svc.svc:9000' } + + before do + stub_dns(host, ip_address: '127.0.0.1') + end + + it 'raises an error' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + + context 'with an IP address' do + let(:host) { 'http://127.0.0.1:9000' } + + it 'raises an error' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + end + end + + context 'when the URL hostname is a domain' do + context 'when domain can be resolved' do + let(:import_url) { 'https://example.org' } + + before do + stub_dns(import_url, ip_address: '93.184.216.34') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'https://93.184.216.34' } + let(:expected_hostname) { 'example.org' } + let(:expected_use_proxy) { false } + end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' + end + + context 'when domain cannot be resolved' do + let(:import_url) { 'http://foobar.x' } + + before do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + end + + it 'raises an error' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + + context 'with HTTP_PROXY' do + let(:import_url) { 'http://foobar.x' } + + before do + stub_env('http_proxy', 'http://proxy.example.com') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { true } + end + + context 'with no_proxy' do + before do + stub_env('no_proxy', 'foobar.x') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } + end + end + end + end + + context 'when domain is too long' do + let(:import_url) { "https://example#{'a' * 1024}.com" } + + it 'raises an error' do + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + end + + context 'when the URL hostname is an IP address' do + let(:import_url) { 'https://93.184.216.34' } + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } + end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' + + context 'when the address is invalid' do + let(:import_url) { 'http://1.1.1.1.1' } + + it 'raises an error' do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + + expect { subject }.to raise_error(described_class::BlockedUrlError) + end + end + end + + context 'when DNS rebinding protection with IP allowed' do + let(:import_url) { 'http://a.192.168.0.120.3times.127.0.0.1.1time.repeat.rebind.network:9121/scrape?target=unix:///var/opt/gitlab/redis/redis.socket&check-keys=*' } + + before do + stub_dns(import_url, ip_address: '192.168.0.120') + + allow(Gitlab::HTTP_V2::UrlAllowlist).to receive(:ip_allowed?).and_return(true) + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'http://192.168.0.120:9121/scrape?target=unix:///var/opt/gitlab/redis/redis.socket&check-keys=*' } + let(:expected_hostname) { 'a.192.168.0.120.3times.127.0.0.1.1time.repeat.rebind.network' } + let(:expected_use_proxy) { false } + end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' + + context 'with HTTP_PROXY' do + before do + stub_env('http_proxy', 'http://proxy.example.com') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { true } + end + + context 'when domain is in no_proxy env' do + before do + stub_env('no_proxy', 'a.192.168.0.120.3times.127.0.0.1.1time.repeat.rebind.network') + end + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { 'http://192.168.0.120:9121/scrape?target=unix:///var/opt/gitlab/redis/redis.socket&check-keys=*' } + let(:expected_hostname) { 'a.192.168.0.120.3times.127.0.0.1.1time.repeat.rebind.network' } + let(:expected_use_proxy) { false } + end + end + end + end + + context 'with disabled DNS rebinding protection' do + let(:options) { { dns_rebind_protection: false, schemes: schemes } } + + context 'when URI is internal' do + let(:import_url) { 'http://localhost' } + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } + end + + it_behaves_like 'a URI exempt from `deny_all_requests_except_allowed`' + end + + context 'when the URL hostname is a domain' do + let(:import_url) { 'https://example.org' } + + before do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + end + + context 'when domain can be resolved' do + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } + end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' + end + + context 'when domain cannot be resolved' do + let(:import_url) { 'http://foobar.x' } + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } + end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' + end + end + + context 'when the URL hostname is an IP address' do + let(:import_url) { 'https://93.184.216.34' } + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } + end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' + + context 'when it is invalid' do + let(:import_url) { 'http://1.1.1.1.1' } + + it_behaves_like 'validates URI and hostname' do + let(:expected_uri) { import_url } + let(:expected_hostname) { nil } + let(:expected_use_proxy) { false } + end + + it_behaves_like 'a URI denied by `deny_all_requests_except_allowed`' + end + end + end + end + + describe '#blocked_url?' do + let(:ports) { [80, 443] } + + it 'allows imports from configured web host and port' do + import_url = "http://localhost:80/t.git" + expect(described_class.blocked_url?(import_url, schemes: schemes)).to be false + end + + it 'allows mirroring from configured SSH host and port' do + import_url = "ssh://localhost:22/t.git" + expect(described_class.blocked_url?(import_url, schemes: schemes)).to be false + end + + it 'returns true for bad localhost hostname' do + expect(described_class.blocked_url?('https://localhost:65535/foo/foo.git', schemes: schemes)).to be true + end + + it 'returns true for bad port' do + expect(described_class.blocked_url?('https://gitlab.com:25/foo/foo.git', ports: ports, schemes: schemes)) + .to be true + end + + it 'returns true for bad scheme' do + expect(described_class.blocked_url?('https://gitlab.com/foo/foo.git', schemes: ['https'])).to be false + expect(described_class.blocked_url?('https://gitlab.com/foo/foo.git', schemes: ['http'])).to be true + end + + it 'returns true for bad protocol on configured web/SSH host and ports' do + web_url = "javascript://localhost:80/t.git%0aalert(1)" + expect(described_class.blocked_url?(web_url, schemes: schemes)).to be true + + ssh_url = "javascript://localhost:22/t.git%0aalert(1)" + expect(described_class.blocked_url?(ssh_url, schemes: schemes)).to be true + end + + it 'returns true for localhost IPs' do + expect(described_class.blocked_url?('https://[0:0:0:0:0:0:0:0]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://0.0.0.0/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::]/foo/foo.git', schemes: schemes)).to be true + end + + it 'returns true for loopback IP' do + expect(described_class.blocked_url?('https://127.0.0.2/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://127.0.0.1/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::1]/foo/foo.git', schemes: schemes)).to be true + end + + it 'returns true for alternative version of 127.0.0.1 (0177.1)' do + expect(described_class.blocked_url?('https://0177.1:65535/foo/foo.git', schemes: schemes)).to be true + end + + it 'returns true for alternative version of 127.0.0.1 (017700000001)' do + expect(described_class.blocked_url?('https://017700000001:65535/foo/foo.git', schemes: schemes)).to be true + end + + it 'returns true for alternative version of 127.0.0.1 (0x7f.1)' do + expect(described_class.blocked_url?('https://0x7f.1:65535/foo/foo.git', schemes: schemes)).to be true + end + + it 'returns true for alternative version of 127.0.0.1 (0x7f.0.0.1)' do + expect(described_class.blocked_url?('https://0x7f.0.0.1:65535/foo/foo.git', schemes: schemes)).to be true + end + + it 'returns true for alternative version of 127.0.0.1 (0x7f000001)' do + expect(described_class.blocked_url?('https://0x7f000001:65535/foo/foo.git', schemes: schemes)).to be true + end + + it 'returns true for alternative version of 127.0.0.1 (2130706433)' do + expect(described_class.blocked_url?('https://2130706433:65535/foo/foo.git', schemes: schemes)).to be true + end + + it 'returns true for alternative version of 127.0.0.1 (127.000.000.001)' do + expect(described_class.blocked_url?('https://127.000.000.001:65535/foo/foo.git', schemes: schemes)).to be true + end + + it 'returns true for alternative version of 127.0.0.1 (127.0.1)' do + expect(described_class.blocked_url?('https://127.0.1:65535/foo/foo.git', schemes: schemes)).to be true + end + + context 'with ipv6 mapped address' do + it 'returns true for localhost IPs' do + expect(described_class.blocked_url?('https://[0:0:0:0:0:ffff:0.0.0.0]/foo/foo.git', schemes: schemes)) + .to be true + expect(described_class.blocked_url?('https://[::ffff:0.0.0.0]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::ffff:0:0]/foo/foo.git', schemes: schemes)).to be true + end + + it 'returns true for loopback IPs' do + expect(described_class.blocked_url?('https://[0:0:0:0:0:ffff:127.0.0.1]/foo/foo.git', schemes: schemes)) + .to be true + expect(described_class.blocked_url?('https://[::ffff:127.0.0.1]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::ffff:7f00:1]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[0:0:0:0:0:ffff:127.0.0.2]/foo/foo.git', schemes: schemes)) + .to be true + expect(described_class.blocked_url?('https://[::ffff:127.0.0.2]/foo/foo.git', schemes: schemes)).to be true + expect(described_class.blocked_url?('https://[::ffff:7f00:2]/foo/foo.git', schemes: schemes)).to be true + end + end + + it 'returns true for a non-alphanumeric hostname' do + aggregate_failures do + expect(described_class).to be_blocked_url('ssh://-oProxyCommand=whoami/a', schemes: ['ssh']) + + # The leading character here is a Unicode "soft hyphen" + expect(described_class).to be_blocked_url('ssh://oProxyCommand=whoami/a', schemes: ['ssh']) + + # Unicode alphanumerics are allowed + expect(described_class).not_to be_blocked_url('ssh://ğitlab.com/a', schemes: ['ssh']) + end + end + + it 'returns true for invalid URL' do + expect(described_class.blocked_url?('http://:8080', schemes: schemes)).to be true + end + + it 'returns false for legitimate URL' do + expect(described_class.blocked_url?('https://gitlab.com/foo/foo.git', schemes: schemes)).to be false + end + + describe 'allow_local_network' do + let(:shared_address_space_ips) { ['100.64.0.0', '100.64.127.127', '100.64.255.255'] } + + let(:local_ips) do + [ + '192.168.1.2', + '[0:0:0:0:0:ffff:192.168.1.2]', + '[::ffff:c0a8:102]', + '10.0.0.2', + '[0:0:0:0:0:ffff:10.0.0.2]', + '[::ffff:a00:2]', + '172.16.0.2', + '[0:0:0:0:0:ffff:172.16.0.2]', + '[::ffff:ac10:20]', + '[feef::1]', + '[fee2::]', + '[fc00:bf8b:e62c:abcd:abcd:aaaa:aaaa:aaaa]', + *shared_address_space_ips + ] + end + + let(:limited_broadcast_address_variants) do + [ + '255.255.255.255', # "normal" dotted decimal + '0377.0377.0377.0377', # Octal + '0377.00000000377.00377.0000377', # Still octal + '0xff.0xff.0xff.0xff', # hex + '0xffffffff', # still hex + '0xBaaaaaaaaaaaaaaaaffffffff', # padded hex + '255.255.255.255:65535', # with a port + '4294967295', # as an integer / dword + '[::ffff:ffff:ffff]', # short IPv6 + '[0000:0000:0000:0000:0000:ffff:ffff:ffff]' # long IPv6 + ] + end + + let(:fake_domain) { 'www.fakedomain.fake' } + + shared_examples 'allows local requests' do + it 'does not block urls from private networks' do + local_ips.each do |ip| + stub_domain_resolv(fake_domain, ip) do + expect(described_class).not_to be_blocked_url("http://#{fake_domain}", **url_blocker_attributes) + end + + expect(described_class).not_to be_blocked_url("http://#{ip}", **url_blocker_attributes) + end + end + + it 'allows localhost endpoints' do + expect(described_class).not_to be_blocked_url('http://0.0.0.0', **url_blocker_attributes) + expect(described_class).not_to be_blocked_url('http://localhost', **url_blocker_attributes) + expect(described_class).not_to be_blocked_url('http://127.0.0.1', **url_blocker_attributes) + end + + it 'allows loopback endpoints' do + expect(described_class).not_to be_blocked_url('http://127.0.0.2', **url_blocker_attributes) + end + + it 'allows IPv4 link-local endpoints' do + expect(described_class).not_to be_blocked_url('http://169.254.169.254', **url_blocker_attributes) + expect(described_class).not_to be_blocked_url('http://169.254.168.100', **url_blocker_attributes) + end + + it 'allows IPv6 link-local endpoints' do + expect(described_class).not_to be_blocked_url( + 'http://[0:0:0:0:0:ffff:169.254.169.254]', **url_blocker_attributes) + expect(described_class).not_to be_blocked_url('http://[::ffff:169.254.169.254]', **url_blocker_attributes) + expect(described_class).not_to be_blocked_url('http://[::ffff:a9fe:a9fe]', **url_blocker_attributes) + expect(described_class).not_to be_blocked_url( + 'http://[0:0:0:0:0:ffff:169.254.168.100]', **url_blocker_attributes) + expect(described_class).not_to be_blocked_url('http://[::ffff:169.254.168.100]', **url_blocker_attributes) + expect(described_class).not_to be_blocked_url('http://[::ffff:a9fe:a864]', **url_blocker_attributes) + expect(described_class).not_to be_blocked_url('http://[fe80::c800:eff:fe74:8]', **url_blocker_attributes) + end + + it 'allows limited broadcast address 255.255.255.255 and variants' do + limited_broadcast_address_variants.each do |variant| + expect(described_class).not_to be_blocked_url("https://#{variant}", **url_blocker_attributes), + "Expected #{variant} to be allowed" + end + end + end + + context 'when true (default)' do + let(:url_blocker_attributes) do + options.merge( + allow_localhost: true, + allow_local_network: true + ) + end + + let(:options) { { schemes: schemes } } + + it_behaves_like 'allows local requests', + { allow_localhost: true, allow_local_network: true, schemes: %w[http https] } + end + + context 'when false' do + it 'blocks urls from private networks' do + local_ips.each do |ip| + stub_domain_resolv(fake_domain, ip) do + expect(described_class).to be_blocked_url( + "http://#{fake_domain}", allow_local_network: false, schemes: schemes) + end + + expect(described_class).to be_blocked_url("http://#{ip}", allow_local_network: false, schemes: schemes) + end + end + + it 'blocks IPv4 link-local endpoints' do + expect(described_class).to be_blocked_url( + 'http://169.254.169.254', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url( + 'http://169.254.168.100', allow_local_network: false, schemes: schemes) + end + + it 'blocks IPv6 link-local endpoints' do + expect(described_class).to be_blocked_url( + 'http://[0:0:0:0:0:ffff:169.254.169.254]', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url( + 'http://[::ffff:169.254.169.254]', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url( + 'http://[::ffff:a9fe:a9fe]', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url( + 'http://[0:0:0:0:0:ffff:169.254.168.100]', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url( + 'http://[::ffff:169.254.168.100]', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url( + 'http://[::ffff:a9fe:a864]', allow_local_network: false, schemes: schemes) + expect(described_class).to be_blocked_url( + 'http://[fe80::c800:eff:fe74:8]', allow_local_network: false, schemes: schemes) + end + + it 'blocks limited broadcast address 255.255.255.255 and variants' do + # Raise BlockedUrlError for invalid URLs. + # The padded hex version, for example, is a valid URL on Mac but + # not on Ubuntu. + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + + limited_broadcast_address_variants.each do |variant| + expect(described_class).to be_blocked_url( + "https://#{variant}", allow_local_network: false, schemes: schemes), + "Expected #{variant} to be blocked" + end + end + + context 'when local domain/IP is allowed' do + let(:url_blocker_attributes) do + options.merge( + allow_localhost: false, + allow_local_network: false + ) + end + + let(:options) { { schemes: schemes, outbound_local_requests_allowlist: allowlist } } + + context 'with IPs in allowlist' do + let(:allowlist) do + [ + '0.0.0.0', + '127.0.0.1', + '127.0.0.2', + '192.168.1.1', + *local_ips, + '0:0:0:0:0:ffff:169.254.169.254', + '::ffff:a9fe:a9fe', + '::ffff:169.254.168.100', + '::ffff:a9fe:a864', + 'fe80::c800:eff:fe74:8', + '255.255.255.255', + + # garbage IPs + '45645632345', + 'garbage456:more345gar:bage' + ] + end + + it_behaves_like 'allows local requests', + { allow_localhost: false, allow_local_network: false, schemes: %w[http https] } + + it 'allows IP when dns_rebind_protection is disabled' do + url = "http://example.com" + attrs = url_blocker_attributes.merge(dns_rebind_protection: false) + + stub_domain_resolv('example.com', '192.168.1.2') do + expect(described_class).not_to be_blocked_url(url, **attrs) + end + + stub_domain_resolv('example.com', '192.168.1.3') do + expect(described_class).to be_blocked_url(url, **attrs) + end + end + + it 'allows the limited broadcast address 255.255.255.255' do + expect(described_class).not_to be_blocked_url('http://255.255.255.255', **url_blocker_attributes) + end + end + + context 'with domains in allowlist' do + let(:allowlist) do + [ + 'www.example.com', + 'example.com', + 'xn--itlab-j1a.com', + 'garbage$^$%#$^&$' + ] + end + + it 'allows domains present in allowlist' do + domain = 'example.com' + subdomain1 = 'www.example.com' + subdomain2 = 'subdomain.example.com' + + stub_domain_resolv(domain, '192.168.1.1') do + expect(described_class).not_to be_blocked_url("http://#{domain}", + **url_blocker_attributes) + end + + stub_domain_resolv(subdomain1, '192.168.1.1') do + expect(described_class).not_to be_blocked_url("http://#{subdomain1}", + **url_blocker_attributes) + end + + # subdomain2 is not part of the allowlist so it should be blocked + stub_domain_resolv(subdomain2, '192.168.1.1') do + expect(described_class).to be_blocked_url("http://#{subdomain2}", + **url_blocker_attributes) + end + end + + it 'works with unicode and idna encoded domains' do + unicode_domain = 'ğitlab.com' + idna_encoded_domain = 'xn--itlab-j1a.com' + + stub_domain_resolv(unicode_domain, '192.168.1.1') do + expect(described_class).not_to be_blocked_url("http://#{unicode_domain}", + **url_blocker_attributes) + end + + stub_domain_resolv(idna_encoded_domain, '192.168.1.1') do + expect(described_class).not_to be_blocked_url("http://#{idna_encoded_domain}", + **url_blocker_attributes) + end + end + + shared_examples 'dns rebinding checks' do + shared_examples 'allowlists the domain' do + let(:allowlist) { [domain] } + let(:url) { "http://#{domain}" } + + before do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + end + + it do + expect(described_class).not_to be_blocked_url(url, **options, dns_rebind_protection: dns_rebind_value) + end + end + + describe 'dns_rebinding_setting' do + context 'when enabled' do + let(:dns_rebind_value) { true } + + it_behaves_like 'allowlists the domain' + end + + context 'when disabled' do + let(:dns_rebind_value) { false } + + it_behaves_like 'allowlists the domain' + end + end + end + + context 'when the domain cannot be resolved' do + let(:domain) { 'foobar.x' } + + it_behaves_like 'dns rebinding checks' + end + + context 'when the domain can be resolved' do + let(:domain) { 'example.com' } + + before do + stub_dns(url, ip_address: '93.184.216.34') + end + + it_behaves_like 'dns rebinding checks' + end + end + + context 'with ports' do + let(:allowlist) do + ["127.0.0.1:2000"] + end + + it 'allows domain with port when resolved ip has port allowed' do + stub_domain_resolv("www.resolve-domain.com", '127.0.0.1', 2000) do + expect(described_class).not_to be_blocked_url( + "http://www.resolve-domain.com:2000", **url_blocker_attributes) + end + end + end + end + end + end + + describe 'enforce_user' do + context 'when false (default)' do + it 'does not block urls with a non-alphanumeric username' do + expect(described_class).not_to be_blocked_url('ssh://-oProxyCommand=whoami@example.com/a', schemes: ['ssh']) + + # The leading character here is a Unicode "soft hyphen" + expect(described_class).not_to be_blocked_url('ssh://oProxyCommand=whoami@example.com/a', schemes: ['ssh']) + + # Unicode alphanumerics are allowed + expect(described_class).not_to be_blocked_url('ssh://ğitlab@example.com/a', schemes: ['ssh']) + end + end + + context 'when true' do + it 'blocks urls with a non-alphanumeric username' do + aggregate_failures do + expect(described_class).to be_blocked_url( + 'ssh://-oProxyCommand=whoami@example.com/a', enforce_user: true, schemes: ['ssh']) + + # The leading character here is a Unicode "soft hyphen" + expect(described_class).to be_blocked_url( + 'ssh://oProxyCommand=whoami@example.com/a', enforce_user: true, schemes: ['ssh']) + + # Unicode alphanumerics are allowed + expect(described_class).not_to be_blocked_url( + 'ssh://ğitlab@example.com/a', enforce_user: true, schemes: ['ssh']) + end + end + end + end + + context 'when ascii_only is true' do + it 'returns true for unicode domain' do + expect(described_class.blocked_url?( + 'https://𝕘itⅼαƄ.com/foo/foo.bar', ascii_only: true, schemes: schemes)).to be true + end + + it 'returns true for unicode tld' do + expect(described_class.blocked_url?( + 'https://gitlab.ᴄοｍ/foo/foo.bar', ascii_only: true, schemes: schemes)).to be true + end + + it 'returns true for unicode path' do + expect(described_class.blocked_url?( + 'https://gitlab.com/𝒇οο/𝒇οο.Ƅαꮁ', ascii_only: true, schemes: schemes)).to be true + end + + it 'returns true for IDNA deviations' do + expect(described_class.blocked_url?( + 'https://mißile.com/foo/foo.bar', ascii_only: true, schemes: schemes)).to be true + expect(described_class.blocked_url?( + 'https://miςςile.com/foo/foo.bar', ascii_only: true, schemes: schemes)).to be true + expect(described_class.blocked_url?( + 'https://gitlab.com/foo/foo.bar', ascii_only: true, schemes: schemes)).to be true + expect(described_class.blocked_url?( + 'https://gitlab.com/foo/foo.bar', ascii_only: true, schemes: schemes)).to be true + end + end + + it 'blocks urls with invalid ip address' do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + + expect(described_class).to be_blocked_url('http://8.8.8.8.8', schemes: schemes) + end + + it 'blocks urls whose hostname cannot be resolved' do + stub_env('RSPEC_ALLOW_INVALID_URLS', 'false') + + expect(described_class).to be_blocked_url('http://foobar.x', schemes: schemes) + end + + context 'when gitlab is running on a non-default port' do + let(:gitlab_port) { 3000 } + + before do + Gitlab::HTTP_V2.configuration.allowed_internal_uris = [ + URI::HTTP.build( + scheme: 'http', + host: 'gitlab.local', + port: gitlab_port + ) + ] + end + + it 'returns true for url targeting the wrong port' do + stub_domain_resolv('gitlab.local', '127.0.0.1') do + expect(described_class).to be_blocked_url("http://gitlab.local/foo", schemes: schemes) + end + end + + it 'does not block url on gitlab port' do + stub_domain_resolv('gitlab.local', '127.0.0.1', gitlab_port) do + expect(described_class).not_to be_blocked_url("http://gitlab.local:#{gitlab_port}/foo", schemes: schemes) + end + end + end + + def stub_domain_resolv(domain, ip, port = 80) + address = instance_double(Addrinfo, + ip_address: ip, + ipv4_private?: true, + ipv6_linklocal?: false, + ipv4_loopback?: false, + ipv6_loopback?: false, + ipv4?: false, + ip_port: port + ) + allow(Addrinfo).to receive(:getaddrinfo).with(domain, port, any_args).and_return([address]) + allow(address).to receive(:ipv6_v4mapped?).and_return(false) + + yield + + allow(Addrinfo).to receive(:getaddrinfo).and_call_original + end + end + + describe '#validate_hostname' do + let(:ip_addresses) do + [ + '2001:db8:1f70::999:de8:7648:6e8', + 'FE80::C800:EFF:FE74:8', + '::ffff:127.0.0.1', + '::ffff:169.254.168.100', + '::ffff:7f00:1', + '0:0:0:0:0:ffff:0.0.0.0', + 'localhost', + '127.0.0.1', + '127.000.000.001', + '0x7f000001', + '0x7f.0.0.1', + '0x7f.0.0.1', + '017700000001', + '0177.1', + '2130706433', + '::', + '::1' + ] + end + + it 'does not raise error for valid Ip addresses' do + ip_addresses.each do |ip| + expect { described_class.send(:validate_hostname, ip) }.not_to raise_error + end + end + end +end diff --git a/gems/gitlab-http/spec/gitlab/http_v2_spec.rb b/gems/gitlab-http/spec/gitlab/http_v2_spec.rb new file mode 100644 index 00000000000..bfa1dcd2633 --- /dev/null +++ b/gems/gitlab-http/spec/gitlab/http_v2_spec.rb @@ -0,0 +1,453 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::HTTP_V2, feature_category: :shared do + context 'when allow_local_requests' do + it 'sends the request to the correct URI' do + stub_full_request('https://example.org:8080', ip_address: '8.8.8.8').to_return(status: 200) + + described_class.get('https://example.org:8080', allow_local_requests: false) + + expect(WebMock).to have_requested(:get, 'https://8.8.8.8:8080').once + end + end + + context 'when not allow_local_requests' do + it 'sends the request to the correct URI' do + stub_full_request('https://example.org:8080') + + described_class.get('https://example.org:8080', allow_local_requests: true) + + expect(WebMock).to have_requested(:get, 'https://8.8.8.9:8080').once + end + end + + context 'when reading the response is too slow' do + before(:all) do + # Override Net::HTTP to add a delay between sending each response chunk + mocked_http = Class.new(Net::HTTP) do + def request(*) + super do |response| + response.instance_eval do + def read_body(*) + mock_stream = @body.split(' ') + mock_stream.each do |fragment| + sleep 0.002.seconds + + yield fragment if block_given? + end + + @body + end + end + + yield response if block_given? + + response + end + end + end + + @original_net_http = Net.send(:remove_const, :HTTP) + @webmock_net_http = WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_get(:@webMockNetHTTP) + + Net.send(:const_set, :HTTP, mocked_http) + WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_set(:@webMockNetHTTP, mocked_http) + + # Reload Gitlab::NetHttpAdapter + described_class.send(:remove_const, :NetHttpAdapter) + load "gitlab/http_v2/net_http_adapter.rb" + end + + before do + stub_const("#{described_class}::Client::DEFAULT_READ_TOTAL_TIMEOUT", 0.001.seconds) + + WebMock.stub_request(:post, /.*/).to_return do + { body: "chunk-1 chunk-2", status: 200 } + end + end + + after(:all) do + Net.send(:remove_const, :HTTP) + Net.send(:const_set, :HTTP, @original_net_http) # rubocop:disable RSpec/InstanceVariable + WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_set(:@webMockNetHTTP, @webmock_net_http) # rubocop:disable RSpec/InstanceVariable + + # Reload Gitlab::NetHttpAdapter + described_class.send(:remove_const, :NetHttpAdapter) + load "gitlab/http_v2/net_http_adapter.rb" + end + + let(:options) { {} } + + subject(:request_slow_responder) { described_class.post('http://example.org', **options) } + + it 'raises an error' do + expect do + request_slow_responder + end.to raise_error(Gitlab::HTTP_V2::ReadTotalTimeout, + /Request timed out after ?([0-9]*[.])?[0-9]+ seconds/) + end + + context 'and timeout option is greater than DEFAULT_READ_TOTAL_TIMEOUT' do + let(:options) { { timeout: 10.seconds } } + + it 'does not raise an error' do + expect { request_slow_responder }.not_to raise_error + end + end + + context 'and stream_body option is truthy' do + let(:options) { { stream_body: true } } + + it 'does not raise an error' do + expect { request_slow_responder }.not_to raise_error + end + end + end + + it 'calls a block' do + WebMock.stub_request(:post, /.*/) + + expect { |b| described_class.post('http://example.org', &b) }.to yield_with_args + end + + describe 'allow_local_requests' do + before do + WebMock.stub_request(:get, /.*/).to_return(status: 200, body: 'Success') + end + + context 'when it is disabled' do + it 'deny requests to localhost' do + expect do + described_class.get('http://localhost:3003', allow_local_requests: false) + end.to raise_error(Gitlab::HTTP_V2::BlockedUrlError) + end + + it 'deny requests to private network' do + expect do + described_class.get('http://192.168.1.2:3003', allow_local_requests: false) + end.to raise_error(Gitlab::HTTP_V2::BlockedUrlError) + end + + context 'if allow_local_requests set to true' do + it 'override the global value and allow requests to localhost or private network' do + stub_full_request('http://localhost:3003') + + expect { described_class.get('http://localhost:3003', allow_local_requests: true) }.not_to raise_error + end + end + end + + context 'when it is enabled' do + it 'allow requests to localhost' do + stub_full_request('http://localhost:3003') + + expect { described_class.get('http://localhost:3003', allow_local_requests: true) }.not_to raise_error + end + + it 'allow requests to private network' do + expect { described_class.get('http://192.168.1.2:3003', allow_local_requests: true) }.not_to raise_error + end + + context 'if allow_local_requests set to false' do + it 'override the global value and ban requests to localhost or private network' do + expect do + described_class.get('http://localhost:3003', + allow_local_requests: false) + end.to raise_error(Gitlab::HTTP_V2::BlockedUrlError) + end + end + end + end + + describe 'handle redirect loops' do + before do + stub_full_request("http://example.org", method: :any) + .to_raise(HTTParty::RedirectionTooDeep.new("Redirection Too Deep")) + end + + it 'handles GET requests' do + expect { described_class.get('http://example.org') }.to raise_error(Gitlab::HTTP_V2::RedirectionTooDeep) + end + + it 'handles POST requests' do + expect { described_class.post('http://example.org') }.to raise_error(Gitlab::HTTP_V2::RedirectionTooDeep) + end + + it 'handles PUT requests' do + expect { described_class.put('http://example.org') }.to raise_error(Gitlab::HTTP_V2::RedirectionTooDeep) + end + + it 'handles DELETE requests' do + expect { described_class.delete('http://example.org') }.to raise_error(Gitlab::HTTP_V2::RedirectionTooDeep) + end + + it 'handles HEAD requests' do + expect { described_class.head('http://example.org') }.to raise_error(Gitlab::HTTP_V2::RedirectionTooDeep) + end + end + + describe 'setting default timeouts' do + let(:default_timeout_options) { described_class::Client::DEFAULT_TIMEOUT_OPTIONS } + + before do + stub_full_request('http://example.org', method: :any) + end + + context 'when no timeouts are set' do + it 'sets default open and read and write timeouts' do + expect(described_class::Client).to receive(:httparty_perform_request).with( + Net::HTTP::Get, 'http://example.org', default_timeout_options + ).and_call_original + + described_class.get('http://example.org') + end + end + + context 'when :timeout is set' do + it 'does not set any default timeouts' do + expect(described_class::Client).to receive(:httparty_perform_request).with( + Net::HTTP::Get, 'http://example.org', { timeout: 1 } + ).and_call_original + + described_class.get('http://example.org', timeout: 1) + end + end + + context 'when :open_timeout is set' do + it 'only sets default read and write timeout' do + expect(described_class::Client).to receive(:httparty_perform_request).with( + Net::HTTP::Get, 'http://example.org', default_timeout_options.merge(open_timeout: 1) + ).and_call_original + + described_class.get('http://example.org', open_timeout: 1) + end + end + + context 'when :read_timeout is set' do + it 'only sets default open and write timeout' do + expect(described_class::Client).to receive(:httparty_perform_request).with( + Net::HTTP::Get, 'http://example.org', default_timeout_options.merge(read_timeout: 1) + ).and_call_original + + described_class.get('http://example.org', read_timeout: 1) + end + end + + context 'when :write_timeout is set' do + it 'only sets default open and read timeout' do + expect(described_class::Client).to receive(:httparty_perform_request).with( + Net::HTTP::Put, 'http://example.org', default_timeout_options.merge(write_timeout: 1) + ).and_call_original + + described_class.put('http://example.org', write_timeout: 1) + end + end + end + + describe '.try_get' do + let(:path) { 'http://example.org' } + let(:default_timeout_options) { described_class::Client::DEFAULT_TIMEOUT_OPTIONS } + + let(:extra_log_info_proc) do + proc do |error, url, options| + { klass: error.class, url: url, options: options } + end + end + + let(:request_options) do + { + **default_timeout_options, + verify: false, + basic_auth: { username: 'user', password: 'pass' } + } + end + + described_class::HTTP_ERRORS.each do |exception_class| + context "with #{exception_class}" do + let(:klass) { exception_class } + + context 'with path' do + before do + expect(described_class::Client).to receive(:httparty_perform_request) # rubocop:disable RSpec/ExpectInHook + .with(Net::HTTP::Get, path, default_timeout_options) + .and_raise(klass) + end + + it 'handles requests without extra_log_info' do + expect(described_class.configuration) + .to receive(:log_exception) + .with(instance_of(klass), {}) + + expect(described_class.try_get(path)).to be_nil + end + + it 'handles requests with extra_log_info as hash' do + expect(described_class.configuration) + .to receive(:log_exception) + .with(instance_of(klass), { a: :b }) + + expect(described_class.try_get(path, extra_log_info: { a: :b })).to be_nil + end + + it 'handles requests with extra_log_info as proc' do + expect(described_class.configuration) + .to receive(:log_exception) + .with(instance_of(klass), { url: path, klass: klass, options: {} }) + + expect(described_class.try_get(path, extra_log_info: extra_log_info_proc)).to be_nil + end + end + + context 'with path and options' do + before do + expect(described_class::Client).to receive(:httparty_perform_request) # rubocop:disable RSpec/ExpectInHook + .with(Net::HTTP::Get, path, request_options) + .and_raise(klass) + end + + it 'handles requests without extra_log_info' do + expect(described_class.configuration) + .to receive(:log_exception) + .with(instance_of(klass), {}) + + expect(described_class.try_get(path, request_options)).to be_nil + end + + it 'handles requests with extra_log_info as hash' do + expect(described_class.configuration) + .to receive(:log_exception) + .with(instance_of(klass), { a: :b }) + + expect(described_class.try_get(path, **request_options, extra_log_info: { a: :b })).to be_nil + end + + it 'handles requests with extra_log_info as proc' do + expect(described_class.configuration) + .to receive(:log_exception) + .with(instance_of(klass), { klass: klass, url: path, options: request_options }) + + expect(described_class.try_get(path, **request_options, extra_log_info: extra_log_info_proc)).to be_nil + end + end + + context 'with path, options, and block' do + let(:block) do + proc {} + end + + before do + expect(described_class::Client).to receive(:httparty_perform_request) # rubocop:disable RSpec/ExpectInHook + .with(Net::HTTP::Get, path, request_options, &block) + .and_raise(klass) + end + + it 'handles requests without extra_log_info' do + expect(described_class.configuration) + .to receive(:log_exception) + .with(instance_of(klass), {}) + + expect(described_class.try_get(path, request_options, &block)).to be_nil + end + + it 'handles requests with extra_log_info as hash' do + expect(described_class.configuration) + .to receive(:log_exception) + .with(instance_of(klass), { a: :b }) + + expect(described_class.try_get(path, **request_options, extra_log_info: { a: :b }, &block)).to be_nil + end + + it 'handles requests with extra_log_info as proc' do + expect(described_class.configuration) + .to receive(:log_exception) + .with(instance_of(klass), { klass: klass, url: path, options: request_options }) + + expect( + described_class.try_get(path, **request_options, extra_log_info: extra_log_info_proc, &block) + ).to be_nil + end + end + end + end + end + + describe 'silent mode', feature_category: :geo_replication do + before do + stub_full_request("http://example.org", method: :any) + end + + context 'when silent mode is enabled' do + let(:silent_mode) { true } + + it 'allows GET requests' do + expect { described_class.get('http://example.org', silent_mode_enabled: silent_mode) }.not_to raise_error + end + + it 'allows HEAD requests' do + expect { described_class.head('http://example.org', silent_mode_enabled: silent_mode) }.not_to raise_error + end + + it 'allows OPTIONS requests' do + expect { described_class.options('http://example.org', silent_mode_enabled: silent_mode) }.not_to raise_error + end + + it 'blocks POST requests' do + expect do + described_class.post('http://example.org', silent_mode_enabled: silent_mode) + end.to raise_error(Gitlab::HTTP_V2::SilentModeBlockedError) + end + + it 'blocks PUT requests' do + expect do + described_class.put('http://example.org', silent_mode_enabled: silent_mode) + end.to raise_error(Gitlab::HTTP_V2::SilentModeBlockedError) + end + + it 'blocks DELETE requests' do + expect do + described_class.delete('http://example.org', silent_mode_enabled: silent_mode) + end.to raise_error(Gitlab::HTTP_V2::SilentModeBlockedError) + end + + it 'logs blocked requests' do + expect(described_class.configuration).to receive(:silent_mode_log_info).with( + "Outbound HTTP request blocked", 'Net::HTTP::Post' + ) + + expect do + described_class.post('http://example.org', silent_mode_enabled: silent_mode) + end.to raise_error(Gitlab::HTTP_V2::SilentModeBlockedError) + end + end + + context 'when silent mode is disabled' do + let(:silent_mode) { false } + + it 'allows GET requests' do + expect { described_class.get('http://example.org', silent_mode_enabled: silent_mode) }.not_to raise_error + end + + it 'allows HEAD requests' do + expect { described_class.head('http://example.org', silent_mode_enabled: silent_mode) }.not_to raise_error + end + + it 'allows OPTIONS requests' do + expect { described_class.options('http://example.org', silent_mode_enabled: silent_mode) }.not_to raise_error + end + + it 'blocks POST requests' do + expect { described_class.post('http://example.org', silent_mode_enabled: silent_mode) }.not_to raise_error + end + + it 'blocks PUT requests' do + expect { described_class.put('http://example.org', silent_mode_enabled: silent_mode) }.not_to raise_error + end + + it 'blocks DELETE requests' do + expect { described_class.delete('http://example.org', silent_mode_enabled: silent_mode) }.not_to raise_error + end + end + end +end diff --git a/gems/gitlab-http/spec/gitlab/stub_requests.rb b/gems/gitlab-http/spec/gitlab/stub_requests.rb new file mode 100644 index 00000000000..ea4a6865251 --- /dev/null +++ b/gems/gitlab-http/spec/gitlab/stub_requests.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module StubRequests + IP_ADDRESS_STUB = '8.8.8.9' + + # Fully stubs a request using WebMock class. This class also + # stubs the IP address the URL is translated to (DNS lookup). + # + # It expects the final request to go to the `ip_address` instead the given url. + # That's primarily a DNS rebind attack prevention of Gitlab::HTTP + # (see: Gitlab::HTTP_V2::UrlBlocker). + # + def stub_full_request(url, ip_address: IP_ADDRESS_STUB, port: 80, method: :get) + stub_dns(url, ip_address: ip_address, port: port) + + url = stubbed_hostname(url, hostname: ip_address) + WebMock.stub_request(method, url) + end + + def stub_dns(url, ip_address:, port: 80) + url = parse_url(url) + socket = Socket.sockaddr_in(port, ip_address) + addr = Addrinfo.new(socket) + + # See Gitlab::UrlBlocker + allow(Addrinfo).to receive(:getaddrinfo) + .with(url.hostname, url.port, nil, :STREAM) + .and_return([addr]) + end + + def stub_all_dns(url, ip_address:) + url = URI(url) + port = 80 # arbitarily chosen, does not matter as we are not going to connect + socket = Socket.sockaddr_in(port, ip_address) + addr = Addrinfo.new(socket) + + # See Gitlab::UrlBlocker + allow(Addrinfo).to receive(:getaddrinfo).and_call_original + allow(Addrinfo).to receive(:getaddrinfo) + .with(url.hostname, anything, nil, :STREAM) + .and_return([addr]) + end + + def stubbed_hostname(url, hostname: IP_ADDRESS_STUB) + url = parse_url(url) + url.hostname = hostname + url.to_s + end + + private + + def parse_url(url) + url.is_a?(URI) ? url : URI(url) + end + end +end |