diff options
Diffstat (limited to 'gems/gitlab-http/lib')
18 files changed, 1078 insertions, 0 deletions
diff --git a/gems/gitlab-http/lib/gitlab-http.rb b/gems/gitlab-http/lib/gitlab-http.rb new file mode 100644 index 00000000000..1fc0e16ec9f --- /dev/null +++ b/gems/gitlab-http/lib/gitlab-http.rb @@ -0,0 +1,11 @@ +# rubocop:disable Naming/FileName + +# frozen_string_literal: true + +# When we say gem 'gitlab-http' in Gemfile, bundler will also run require gitlab-http for us and it'd +# resolve the conflict when we call `Gitlab::HTTP_V2.configure` first time. +# See more: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/125024#note_1502698924 + +require_relative 'gitlab/http_v2' + +# rubocop:enable Naming/FileName diff --git a/gems/gitlab-http/lib/gitlab/http_v2.rb b/gems/gitlab-http/lib/gitlab/http_v2.rb new file mode 100644 index 00000000000..8f3ede95530 --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require_relative "http_v2/configuration" +require_relative "http_v2/patches" +require_relative "http_v2/client" + +module Gitlab + module HTTP_V2 + SUPPORTED_HTTP_METHODS = [:get, :try_get, :post, :patch, :put, :delete, :head, :options].freeze + + class << self + delegate(*SUPPORTED_HTTP_METHODS, to: ::Gitlab::HTTP_V2::Client) + + def configuration + @configuration ||= Configuration.new + end + + def configure + yield(configuration) + end + end + end +end diff --git a/gems/gitlab-http/lib/gitlab/http_v2/buffered_io.rb b/gems/gitlab-http/lib/gitlab/http_v2/buffered_io.rb new file mode 100644 index 00000000000..478b9102dec --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2/buffered_io.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'net/http' +require 'webmock' if Rails.env.test? + +# The Ruby 3.2 does change Net protocol. Please see; +# https://github.com/ruby/ruby/blob/ruby_3_2/lib/net/protocol.rb#L194-L206 +# vs https://github.com/ruby/ruby/blob/ruby_3_1/lib/net/protocol.rb#L190-L200 +NET_PROTOCOL_VERSION_0_2_0 = Gem::Version.new(Net::Protocol::VERSION) >= Gem::Version.new('0.2.0') + +module Gitlab + module HTTP_V2 + # Net::BufferedIO is overwritten by webmock but in order to test this class, + # it needs to inherit from the original BufferedIO. + # https://github.com/bblimke/webmock/blob/867f4b290fd133658aa9530cba4ba8b8c52c0d35/lib/webmock/http_lib_adapters/net_http.rb#L266 + parent_class = if const_defined?('WebMock::HttpLibAdapters::NetHttpAdapter::OriginalNetBufferedIO') && + Rails.env.test? + WebMock::HttpLibAdapters::NetHttpAdapter::OriginalNetBufferedIO + else + Net::BufferedIO + end + + class BufferedIo < parent_class + HEADER_READ_TIMEOUT = 20 + + # rubocop: disable Style/RedundantReturn + # rubocop: disable Cop/LineBreakAfterGuardClauses + # rubocop: disable Layout/EmptyLineAfterGuardClause + + # Original method: + # https://github.com/ruby/ruby/blob/cdb7d699d0641e8f081d590d06d07887ac09961f/lib/net/protocol.rb#L190-L200 + def readuntil(terminator, ignore_eof = false, start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)) + if NET_PROTOCOL_VERSION_0_2_0 + offset = @rbuf_offset + begin + until idx = @rbuf.index(terminator, offset) # rubocop:disable Lint/AssignmentInCondition + if (elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) > HEADER_READ_TIMEOUT + raise Gitlab::HTTP_V2::HeaderReadTimeout, + "Request timed out after reading headers for #{elapsed} seconds" + end + + offset = @rbuf.bytesize + rbuf_fill + end + + return rbuf_consume(idx + terminator.bytesize - @rbuf_offset) + rescue EOFError + raise unless ignore_eof + return rbuf_consume(@rbuf.size) + end + else + begin + until idx = @rbuf.index(terminator) # rubocop:disable Lint/AssignmentInCondition + if (elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) > HEADER_READ_TIMEOUT + raise Gitlab::HTTP_V2::HeaderReadTimeout, + "Request timed out after reading headers for #{elapsed} seconds" + end + + rbuf_fill + end + + return rbuf_consume(idx + terminator.size) + rescue EOFError + raise unless ignore_eof + return rbuf_consume(@rbuf.size) + end + end + end + # rubocop: enable Style/RedundantReturn + # rubocop: enable Cop/LineBreakAfterGuardClauses + # rubocop: enable Layout/EmptyLineAfterGuardClause + end + end +end diff --git a/gems/gitlab-http/lib/gitlab/http_v2/client.rb b/gems/gitlab-http/lib/gitlab/http_v2/client.rb new file mode 100644 index 00000000000..8daf19d7351 --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2/client.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +require 'httparty' +require 'net/http' +require 'active_support/all' +require_relative 'new_connection_adapter' +require_relative "exceptions" + +module Gitlab + module HTTP_V2 + class Client + DEFAULT_TIMEOUT_OPTIONS = { + open_timeout: 10, + read_timeout: 20, + write_timeout: 30 + }.freeze + DEFAULT_READ_TOTAL_TIMEOUT = 30.seconds + + SILENT_MODE_ALLOWED_METHODS = [ + Net::HTTP::Get, + Net::HTTP::Head, + Net::HTTP::Options, + Net::HTTP::Trace + ].freeze + + include HTTParty # rubocop:disable Gitlab/HTTParty + + class << self + alias_method :httparty_perform_request, :perform_request + end + + connection_adapter NewConnectionAdapter + + def self.perform_request(http_method, path, options, &block) + raise_if_blocked_by_silent_mode(http_method) if options.delete(:silent_mode_enabled) + + log_info = options.delete(:extra_log_info) + options_with_timeouts = + if !options.has_key?(:timeout) + options.with_defaults(DEFAULT_TIMEOUT_OPTIONS) + else + options + end + + if options[:stream_body] + httparty_perform_request(http_method, path, options_with_timeouts, &block) + else + begin + start_time = nil + read_total_timeout = options.fetch(:timeout, DEFAULT_READ_TOTAL_TIMEOUT) + + httparty_perform_request(http_method, path, options_with_timeouts) do |fragment| + start_time ||= system_monotonic_time + elapsed = system_monotonic_time - start_time + + raise ReadTotalTimeout, "Request timed out after #{elapsed} seconds" if elapsed > read_total_timeout + + yield fragment if block + end + rescue HTTParty::RedirectionTooDeep + raise RedirectionTooDeep + rescue *HTTP_ERRORS => e + extra_info = log_info || {} + extra_info = log_info.call(e, path, options) if log_info.respond_to?(:call) + configuration.log_exception(e, extra_info) + + raise e + end + end + end + + def self.try_get(path, options = {}, &block) + self.get(path, options, &block) # rubocop:disable Style/RedundantSelf + rescue *HTTP_ERRORS + nil + end + + def self.raise_if_blocked_by_silent_mode(http_method) + return if SILENT_MODE_ALLOWED_METHODS.include?(http_method) + + configuration.silent_mode_log_info('Outbound HTTP request blocked', http_method.to_s) + + raise SilentModeBlockedError, 'only get, head, options, and trace methods are allowed in silent mode' + end + + def self.system_monotonic_time + Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second) + end + + def self.configuration + Gitlab::HTTP_V2.configuration + end + end + end +end diff --git a/gems/gitlab-http/lib/gitlab/http_v2/configuration.rb b/gems/gitlab-http/lib/gitlab/http_v2/configuration.rb new file mode 100644 index 00000000000..98b07d0cf27 --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2/configuration.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module HTTP_V2 + class Configuration + attr_accessor :allowed_internal_uris, :log_exception_proc, :silent_mode_log_info_proc + + def log_exception(...) + log_exception_proc&.call(...) + end + + def silent_mode_log_info(...) + silent_mode_log_info_proc&.call(...) + end + end + end +end diff --git a/gems/gitlab-http/lib/gitlab/http_v2/domain_allowlist_entry.rb b/gems/gitlab-http/lib/gitlab/http_v2/domain_allowlist_entry.rb new file mode 100644 index 00000000000..5a08c891184 --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2/domain_allowlist_entry.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module HTTP_V2 + class DomainAllowlistEntry + attr_reader :domain, :port + + def initialize(domain, port: nil) + @domain = domain + @port = port + end + + def match?(requested_domain, requested_port = nil) + return false unless domain == requested_domain + return true if port.nil? + + port == requested_port + end + end + end +end diff --git a/gems/gitlab-http/lib/gitlab/http_v2/exceptions.rb b/gems/gitlab-http/lib/gitlab/http_v2/exceptions.rb new file mode 100644 index 00000000000..5a34d0b9939 --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2/exceptions.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'net/http' + +module Gitlab + module HTTP_V2 + BlockedUrlError = Class.new(StandardError) + RedirectionTooDeep = Class.new(StandardError) + ReadTotalTimeout = Class.new(Net::ReadTimeout) + HeaderReadTimeout = Class.new(Net::ReadTimeout) + SilentModeBlockedError = Class.new(StandardError) + + HTTP_TIMEOUT_ERRORS = [ + Net::OpenTimeout, Net::ReadTimeout, Net::WriteTimeout, Gitlab::HTTP_V2::ReadTotalTimeout + ].freeze + + HTTP_ERRORS = HTTP_TIMEOUT_ERRORS + [ + EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, + Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH, + Gitlab::HTTP_V2::BlockedUrlError, Gitlab::HTTP_V2::RedirectionTooDeep, + Net::HTTPBadResponse + ].freeze + end +end diff --git a/gems/gitlab-http/lib/gitlab/http_v2/ip_allowlist_entry.rb b/gems/gitlab-http/lib/gitlab/http_v2/ip_allowlist_entry.rb new file mode 100644 index 00000000000..ed5a2dba284 --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2/ip_allowlist_entry.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module HTTP_V2 + class IpAllowlistEntry + attr_reader :ip, :port + + # Argument ip should be an IPAddr object + def initialize(ip, port: nil) + @ip = ip + @port = port + end + + def match?(requested_ip, requested_port = nil) + requested_ip = IPAddr.new(requested_ip) if requested_ip.is_a?(String) + + return false unless ip_include?(requested_ip) + return true if port.nil? + + port == requested_port + end + + private + + # Prior to ipaddr v1.2.3, if the allow list were the IPv4 to IPv6 + # mapped address ::ffff:169.254.168.100 and the requested IP were + # 169.254.168.100 or ::ffff:169.254.168.100, the IP would be + # considered in the allow list. However, with + # https://github.com/ruby/ipaddr/pull/31, IPAddr#include? will + # only match if the IP versions are the same. This method + # preserves backwards compatibility if the versions differ by + # checking inclusion by coercing an IPv4 address to its IPv6 + # mapped address. + def ip_include?(requested_ip) + return true if ip.include?(requested_ip) + return ip.include?(requested_ip.ipv4_mapped) if requested_ip.ipv4? && ip.ipv6? + return ip.ipv4_mapped.include?(requested_ip) if requested_ip.ipv6? && ip.ipv4? + + false + end + end + end +end diff --git a/gems/gitlab-http/lib/gitlab/http_v2/net_http_adapter.rb b/gems/gitlab-http/lib/gitlab/http_v2/net_http_adapter.rb new file mode 100644 index 00000000000..c6af2ed6aff --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2/net_http_adapter.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'net/http' +require 'webmock' if Rails.env.test? +require_relative 'buffered_io' + +module Gitlab + module HTTP_V2 + # Webmock overwrites the Net::HTTP#request method with + # https://github.com/bblimke/webmock/blob/867f4b290fd133658aa9530cba4ba8b8c52c0d35/lib/webmock/http_lib_adapters/net_http.rb#L74 + # Net::HTTP#request usually calls Net::HTTP#connect but the Webmock overwrite doesn't. + # This makes sure that, in a test environment, the superclass is the Webmock overwrite. + parent_class = if defined?(WebMock) && Rails.env.test? + WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_get(:@webMockNetHTTP) + else + Net::HTTP + end + + class NetHttpAdapter < parent_class + private + + def connect + result = super + + @socket = BufferedIo.new(@socket.io, + read_timeout: @socket.read_timeout, + write_timeout: @socket.write_timeout, + continue_timeout: @socket.continue_timeout, + debug_output: @socket.debug_output) + + result + end + end + end +end diff --git a/gems/gitlab-http/lib/gitlab/http_v2/new_connection_adapter.rb b/gems/gitlab-http/lib/gitlab/http_v2/new_connection_adapter.rb new file mode 100644 index 00000000000..ee4be97dc6d --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2/new_connection_adapter.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +# This class is part of the Gitlab::HTTP wrapper. It handles local requests and header timeouts +# +# 1. Local requests +# Depending on the value of the global setting allow_local_requests_from_web_hooks_and_services, +# this adapter will allow/block connection to internal IPs and/or urls. +# +# This functionality can be overridden by providing the setting the option +# allow_local_requests = true in the request. For example: +# Gitlab::HTTP.get('http://www.gitlab.com', allow_local_requests: true) +# +# This option will take precedence over the global setting. +# +# 2. Header timeouts +# When the use_read_total_timeout option is used, that means the receiver +# of the HTTP request cannot be trusted. Gitlab::BufferedIo will be used, +# to read header data. It is a modified version of Net::BufferedIO that +# raises a timeout error if reading header data takes too much time. + +require 'httparty' +require_relative 'net_http_adapter' +require_relative 'url_blocker' + +module Gitlab + module HTTP_V2 + class NewConnectionAdapter < HTTParty::ConnectionAdapter + def initialize(...) + super + + @allow_local_requests = options.delete(:allow_local_requests) + @extra_allowed_uris = options.delete(:extra_allowed_uris) + @deny_all_requests_except_allowed = options.delete(:deny_all_requests_except_allowed) + @outbound_local_requests_allowlist = options.delete(:outbound_local_requests_allowlist) + @dns_rebinding_protection_enabled = options.delete(:dns_rebinding_protection_enabled) + end + + def connection + result = validate_url_with_proxy!(uri) + @uri = result.uri + hostname = result.hostname + + http = super + http.hostname_override = hostname if hostname + + unless result.use_proxy + http.proxy_from_env = false + http.proxy_address = nil + end + + net_adapter = NetHttpAdapter.new(http.address, http.port) + + http.instance_variables.each do |variable| + net_adapter.instance_variable_set(variable, http.instance_variable_get(variable)) + end + + net_adapter + end + + private + + def validate_url_with_proxy!(url) + UrlBlocker.validate_url_with_proxy!(url, **url_blocker_options) + rescue UrlBlocker::BlockedUrlError => e + raise HTTP_V2::BlockedUrlError, "URL is blocked: #{e.message}" + end + + def url_blocker_options + { + allow_local_network: @allow_local_requests, + allow_localhost: @allow_local_requests, + extra_allowed_uris: @extra_allowed_uris, + schemes: %w[http https], + deny_all_requests_except_allowed: @deny_all_requests_except_allowed, + outbound_local_requests_allowlist: @outbound_local_requests_allowlist, + dns_rebind_protection: @dns_rebinding_protection_enabled + }.compact + end + end + end +end diff --git a/gems/gitlab-http/lib/gitlab/http_v2/patches.rb b/gems/gitlab-http/lib/gitlab/http_v2/patches.rb new file mode 100644 index 00000000000..3d26fbc6447 --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2/patches.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +require_relative "../../hostname_override_patch" +require_relative "../../net_http/protocol_patch" +require_relative "../../net_http/response_patch" +require_relative "../../httparty/response_patch" diff --git a/gems/gitlab-http/lib/gitlab/http_v2/url_allowlist.rb b/gems/gitlab-http/lib/gitlab/http_v2/url_allowlist.rb new file mode 100644 index 00000000000..6e17315c87d --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2/url_allowlist.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'gitlab/utils/all' +require_relative 'ip_allowlist_entry' +require_relative 'domain_allowlist_entry' + +module Gitlab + module HTTP_V2 + class UrlAllowlist + class << self + def ip_allowed?(ip_string, allowlist, port: nil) + return false if ip_string.blank? + + ip_allowlist, _ = outbound_local_requests_allowlist_arrays(allowlist) + ip_obj = ::Gitlab::Utils.string_to_ip_object(ip_string) + + ip_allowlist.any? do |ip_allowlist_entry| + ip_allowlist_entry.match?(ip_obj, port) + end + end + + def domain_allowed?(domain_string, allowlist, port: nil) + return false if domain_string.blank? + + _, domain_allowlist = outbound_local_requests_allowlist_arrays(allowlist) + + domain_allowlist.any? do |domain_allowlist_entry| + domain_allowlist_entry.match?(domain_string, port) + end + end + + private + + def outbound_local_requests_allowlist_arrays(allowlist) + return [[], []] if allowlist.blank? + + allowlist.reduce([[], []]) do |(ip_allowlist, domain_allowlist), string| + address, port = parse_addr_and_port(string) + + ip_obj = ::Gitlab::Utils.string_to_ip_object(address) + + if ip_obj + ip_allowlist << IpAllowlistEntry.new(ip_obj, port: port) + else + domain_allowlist << DomainAllowlistEntry.new(address, port: port) + end + + [ip_allowlist, domain_allowlist] + end + end + + def parse_addr_and_port(str) + case str + when /\A\[(?<address> .* )\]:(?<port> \d+ )\z/x # string like "[::1]:80" + address = $~[:address] + port = $~[:port] + when /\A(?<address> [^:]+ ):(?<port> \d+ )\z/x # string like "127.0.0.1:80" + address = $~[:address] + port = $~[:port] + else # string with no port number + address = str + port = nil + end + + [address, port&.to_i] + end + end + end + end +end diff --git a/gems/gitlab-http/lib/gitlab/http_v2/url_blocker.rb b/gems/gitlab-http/lib/gitlab/http_v2/url_blocker.rb new file mode 100644 index 00000000000..a794ab2f443 --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2/url_blocker.rb @@ -0,0 +1,409 @@ +# frozen_string_literal: true + +require 'resolv' +require 'ipaddress' +require_relative 'url_allowlist' + +module Gitlab + module HTTP_V2 + class UrlBlocker + BlockedUrlError = Class.new(StandardError) + HTTP_PROXY_ENV_VARS = %w[http_proxy https_proxy HTTP_PROXY HTTPS_PROXY].freeze + + # Result stores the validation result: + # uri - The original URI requested + # hostname - The hostname that should be used to connect. For DNS + # rebinding protection, this will be the resolved IP address of + # the hostname. + # use_proxy - + # If true, this means that the proxy server specified in the + # http_proxy/https_proxy environment variables should be used. + # + # If false, this either means that no proxy server was specified + # or that the hostname in the URL is exempt via the no_proxy + # environment variable. This allows the caller to disable usage + # of a proxy since the IP address may be used to + # connect. Otherwise, Net::HTTP may erroneously compare the IP + # address against the no_proxy list. + Result = Struct.new(:uri, :hostname, :use_proxy) + + class << self + # Validates the given url according to the constraints specified by arguments. + # + # ports - Raises error if the given URL port is not between given ports. + # allow_localhost - Raises error if URL resolves to a localhost IP address and argument is false. + # allow_local_network - Raises error if URL resolves to a link-local address and argument is false. + # extra_allowed_uris - Array of URI objects that are allowed in addition to hostname and IP constraints. + # This parameter is passed in this class when making the HTTP request. + # ascii_only - Raises error if URL has unicode characters and argument is true. + # enforce_user - Raises error if URL user doesn't start with alphanumeric characters and argument is true. + # enforce_sanitization - Raises error if URL includes any HTML/CSS/JS tags and argument is true. + # deny_all_requests_except_allowed - Raises error if URL is not in the allow list and argument is true. + # Can be Boolean or Proc. Defaults to instance app setting. + # dns_rebind_protection - Enforce DNS-rebinding attack protection. + # outbound_local_requests_allowlist - A list of trusted domains or IP addresses to which local requests are + # allowed when local requests for webhooks and integrations are disabled. This parameter is static and + # comes from the `outbound_local_requests_whitelist` application setting. # rubocop:disable Naming/InclusiveLanguage + # + # Returns a Result object. + # rubocop:disable Metrics/ParameterLists + def validate_url_with_proxy!( + url, + schemes:, + ports: [], + allow_localhost: false, + allow_local_network: true, + extra_allowed_uris: [], + ascii_only: false, + enforce_user: false, + enforce_sanitization: false, + deny_all_requests_except_allowed: false, + dns_rebind_protection: true, + outbound_local_requests_allowlist: [] + ) + # rubocop:enable Metrics/ParameterLists + + return Result.new(nil, nil, true) if url.nil? + + raise ArgumentError, 'The schemes is a required argument' if schemes.blank? + + # Param url can be a string, URI or Addressable::URI + uri = parse_url(url) + + validate_uri( + uri: uri, + schemes: schemes, + ports: ports, + enforce_sanitization: enforce_sanitization, + enforce_user: enforce_user, + ascii_only: ascii_only + ) + + begin + address_info = get_address_info(uri) + rescue SocketError + proxy_in_use = uri_under_proxy_setting?(uri, nil) + + unless enforce_address_info_retrievable?(uri, + dns_rebind_protection, + deny_all_requests_except_allowed, + outbound_local_requests_allowlist) + return Result.new(uri, nil, proxy_in_use) + end + + raise BlockedUrlError, 'Host cannot be resolved or invalid' + end + + ip_address = ip_address(address_info) + proxy_in_use = uri_under_proxy_setting?(uri, ip_address) + + # Ignore DNS rebind protection when a proxy is being used, as DNS + # rebinding is expected behavior. + dns_rebind_protection &&= !proxy_in_use + return Result.new(uri, nil, proxy_in_use) if domain_in_allow_list?(uri, outbound_local_requests_allowlist) + + protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, dns_rebind_protection, proxy_in_use) + + if ip_in_allow_list?(ip_address, outbound_local_requests_allowlist, port: get_port(uri)) + return protected_uri_with_hostname + end + + return protected_uri_with_hostname if allowed_uri?(uri, extra_allowed_uris) + + validate_deny_all_requests_except_allowed!(deny_all_requests_except_allowed) + + validate_local_request( + address_info: address_info, + allow_localhost: allow_localhost, + allow_local_network: allow_local_network + ) + + protected_uri_with_hostname + end + + def blocked_url?(url, **kwargs) + validate!(url, **kwargs) + + false + rescue BlockedUrlError + true + end + + # For backwards compatibility, Returns an array with [<uri>, <original-hostname>]. + # Issue for refactoring: https://gitlab.com/gitlab-org/gitlab/-/issues/410890 + def validate!(...) + result = validate_url_with_proxy!(...) + [result.uri, result.hostname] + end + + private + + # Returns the given URI with IP address as hostname and the original hostname respectively + # in an Array. + # + # It checks whether the resolved IP address matches with the hostname. If not, it changes + # the hostname to the resolved IP address. + # + # The original hostname is used to validate the SSL, given in that scenario + # we'll be making the request to the IP address, instead of using the hostname. + def enforce_uri_hostname(ip_address, uri, dns_rebind_protection, proxy_in_use) + unless dns_rebind_protection && ip_address && ip_address != uri.hostname + return Result.new(uri, nil, proxy_in_use) + end + + new_uri = uri.dup + new_uri.hostname = ip_address + Result.new(new_uri, uri.hostname, proxy_in_use) + end + + def ip_address(address_info) + address_info.first&.ip_address + end + + def validate_uri(uri:, schemes:, ports:, enforce_sanitization:, enforce_user:, ascii_only:) + validate_html_tags(uri) if enforce_sanitization + + return if internal?(uri) + + validate_scheme(uri.scheme, schemes) + validate_port(get_port(uri), ports) if ports.any? + validate_user(uri.user) if enforce_user + validate_hostname(uri.hostname) + validate_unicode_restriction(uri) if ascii_only + end + + def uri_under_proxy_setting?(uri, ip_address) + return false unless http_proxy_env? + # `no_proxy|NO_PROXY` specifies addresses for which the proxy is not + # used. If it's empty, there are no exceptions and this URI + # will be under proxy settings. + return true if no_proxy_env.blank? + + # `no_proxy|NO_PROXY` is being used. We must check whether it + # applies to this specific URI. + ::URI::Generic.use_proxy?(uri.hostname, ip_address, get_port(uri), no_proxy_env) + end + + # Returns addrinfo object for the URI. + # + # @param uri [Addressable::URI] + # + # @raise [Gitlab::UrlBlocker::BlockedUrlError, ArgumentError] - BlockedUrlError raised if host is too long. + # + # @return [Array<Addrinfo>] + def get_address_info(uri) + Addrinfo.getaddrinfo(uri.hostname, get_port(uri), nil, :STREAM).map do |addr| + addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr + end + rescue ArgumentError => e + # Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters. + raise unless e.message.include?('hostname too long') + + raise BlockedUrlError, "Host is too long (maximum is 1024 characters)" + end + + def enforce_address_info_retrievable?( + uri, dns_rebind_protection, deny_all_requests_except_allowed, + outbound_local_requests_allowlist) + # Do not enforce if URI is in the allow list + return false if domain_in_allow_list?(uri, outbound_local_requests_allowlist) + + # Enforce if the instance should block requests + return true if deny_all_requests_except_allowed?(deny_all_requests_except_allowed) + + # Do not enforce if DNS rebinding protection is disabled + return false unless dns_rebind_protection + + # Do not enforce if proxy is used + return false if http_proxy_env? + + # In the test suite we use a lot of mocked urls that are either invalid or + # don't exist. In order to avoid modifying a ton of tests and factories + # we allow invalid urls unless the environment variable RSPEC_ALLOW_INVALID_URLS + # is not true + return false if Rails.env.test? && ENV['RSPEC_ALLOW_INVALID_URLS'] == 'true' + + true + end + + def validate_local_request( + address_info:, + allow_localhost:, + allow_local_network:) + return if allow_local_network && allow_localhost + + unless allow_localhost + validate_localhost(address_info) + validate_loopback(address_info) + end + + return if allow_local_network + + validate_local_network(address_info) + validate_link_local(address_info) + validate_shared_address(address_info) + validate_limited_broadcast_address(address_info) + end + + def validate_shared_address(addrs_info) + netmask = IPAddr.new('100.64.0.0/10') + return unless addrs_info.any? { |addr| netmask.include?(addr.ip_address) } + + raise BlockedUrlError, "Requests to the shared address space are not allowed" + end + + def validate_html_tags(uri) + uri_str = uri.to_s + sanitized_uri = ActionController::Base.helpers.sanitize(uri_str, tags: []) + return if sanitized_uri == uri_str + + raise BlockedUrlError, 'HTML/CSS/JS tags are not allowed' + end + + def parse_url(url) + Addressable::URI.parse(url).tap do |parsed_url| + raise Addressable::URI::InvalidURIError if multiline_blocked?(parsed_url) + end + rescue Addressable::URI::InvalidURIError, URI::InvalidURIError + raise BlockedUrlError, 'URI is invalid' + end + + def multiline_blocked?(parsed_url) + url = parsed_url.to_s + + return true if url =~ /\n|\r/ + # Google Cloud Storage uses a multi-line, encoded Signature query string + return false if %w[http https].include?(parsed_url.scheme&.downcase) + + CGI.unescape(url) =~ /\n|\r/ + end + + def validate_port(port, ports) + return if port.blank? + # Only ports under 1024 are restricted + return if port >= 1024 + return if ports.include?(port) + + raise BlockedUrlError, "Only allowed ports are #{ports.join(', ')}, and any over 1024" + end + + def validate_scheme(scheme, schemes) + return unless scheme.blank? || (schemes.any? && schemes.exclude?(scheme)) + + raise BlockedUrlError, "Only allowed schemes are #{schemes.join(', ')}" + end + + def validate_user(value) + return if value.blank? + return if value =~ /\A\p{Alnum}/ + + raise BlockedUrlError, "Username needs to start with an alphanumeric character" + end + + def validate_hostname(value) + return if value.blank? + return if IPAddress.valid?(value) + return if value =~ /\A\p{Alnum}/ + + raise BlockedUrlError, "Hostname or IP address invalid" + end + + def validate_unicode_restriction(uri) + return if uri.to_s.ascii_only? + + raise BlockedUrlError, "URI must be ascii only #{uri.to_s.dump}" + end + + def validate_localhost(addrs_info) + local_ips = ["::", "0.0.0.0"] + local_ips.concat(Socket.ip_address_list.map(&:ip_address)) + + return if (local_ips & addrs_info.map(&:ip_address)).empty? + + raise BlockedUrlError, "Requests to localhost are not allowed" + end + + def validate_loopback(addrs_info) + return unless addrs_info.any? { |addr| addr.ipv4_loopback? || addr.ipv6_loopback? } + + raise BlockedUrlError, "Requests to loopback addresses are not allowed" + end + + def validate_local_network(addrs_info) + return unless addrs_info.any? { |addr| addr.ipv4_private? || addr.ipv6_sitelocal? || addr.ipv6_unique_local? } + + raise BlockedUrlError, "Requests to the local network are not allowed" + end + + def validate_link_local(addrs_info) + netmask = IPAddr.new('169.254.0.0/16') + return unless addrs_info.any? { |addr| addr.ipv6_linklocal? || netmask.include?(addr.ip_address) } + + raise BlockedUrlError, "Requests to the link local network are not allowed" + end + + # Raises a BlockedUrlError if the instance is configured to deny all requests. + # + # This should only be called after allow list checks have been made. + def validate_deny_all_requests_except_allowed!(should_deny) + return unless deny_all_requests_except_allowed?(should_deny) + + raise BlockedUrlError, "Requests to hosts and IP addresses not on the Allow List are denied" + end + + # Raises a BlockedUrlError if any IP in `addrs_info` is the limited + # broadcast address. + # https://datatracker.ietf.org/doc/html/rfc919#section-7 + def validate_limited_broadcast_address(addrs_info) + blocked_ips = ["255.255.255.255"] + + return if (blocked_ips & addrs_info.map(&:ip_address)).empty? + + raise BlockedUrlError, "Requests to the limited broadcast address are not allowed" + end + + def allowed_uri?(uri, extra_allowed_uris) + internal?(uri) || check_uri(uri, extra_allowed_uris) + end + + # Allow url from the GitLab instance itself but only for the configured hostname and ports + def internal?(uri) + check_uri(uri, Gitlab::HTTP_V2.configuration.allowed_internal_uris) + end + + def check_uri(uri, allowlist) + allowlist.any? do |allowed_uri| + allowed_uri.scheme == uri.scheme && + allowed_uri.hostname == uri.hostname && + get_port(allowed_uri) == get_port(uri) + end + end + + def deny_all_requests_except_allowed?(should_deny) + should_deny.is_a?(Proc) ? should_deny.call : should_deny + end + + def domain_in_allow_list?(uri, outbound_local_requests_allowlist) + Gitlab::HTTP_V2::UrlAllowlist.domain_allowed?( + uri.normalized_host, outbound_local_requests_allowlist, port: get_port(uri)) + end + + def ip_in_allow_list?(ip_address, outbound_local_requests_allowlist, port: nil) + Gitlab::HTTP_V2::UrlAllowlist.ip_allowed?(ip_address, outbound_local_requests_allowlist, port: port) + end + + def no_proxy_env + ENV['no_proxy'] || ENV['NO_PROXY'] + end + + def http_proxy_env? + HTTP_PROXY_ENV_VARS.any? { |name| ENV[name].present? } + end + + def get_port(uri) + uri.port || uri.default_port + end + end + end + end +end diff --git a/gems/gitlab-http/lib/gitlab/http_v2/version.rb b/gems/gitlab-http/lib/gitlab/http_v2/version.rb new file mode 100644 index 00000000000..8a9a17de112 --- /dev/null +++ b/gems/gitlab-http/lib/gitlab/http_v2/version.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module HTTP_V2 + module Version + VERSION = "0.1.0" + end + end +end diff --git a/gems/gitlab-http/lib/hostname_override_patch.rb b/gems/gitlab-http/lib/hostname_override_patch.rb new file mode 100644 index 00000000000..c5799bf0682 --- /dev/null +++ b/gems/gitlab-http/lib/hostname_override_patch.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +# This override allows passing `@hostname_override` to the SNI protocol, +# which is used to lookup the correct SSL certificate in the +# request handshake process. +# +# Given we've forced the HTTP request to be sent to the resolved +# IP address in a few scenarios (e.g.: `Gitlab::HTTP_V2` through +# `UrlBlocker.validate!`), we need to provide the _original_ +# hostname via SNI in order to have a clean connection setup. +# +# This is ultimately needed in order to avoid DNS rebinding attacks +# through HTTP requests. + +require 'net/http' + +class OpenSSL::SSL::SSLContext + attr_accessor :hostname_override +end + +class OpenSSL::SSL::SSLSocket + module HostnameOverride + # rubocop: disable Gitlab/ModuleWithInstanceVariables + def hostname=(hostname) + super(@context.hostname_override || hostname) + end + + def post_connection_check(hostname) + super(@context.hostname_override || hostname) + end + # rubocop: enable Gitlab/ModuleWithInstanceVariables + end + + prepend HostnameOverride +end + +class Net::HTTP + attr_accessor :hostname_override + + SSL_IVNAMES << :@hostname_override + SSL_ATTRIBUTES << :hostname_override + + module HostnameOverride + def addr_port + return super unless hostname_override + + addr = hostname_override + default_port = use_ssl? ? Net::HTTP.https_default_port : Net::HTTP.http_default_port + default_port == port ? addr : "#{addr}:#{port}" + end + end + + prepend HostnameOverride +end diff --git a/gems/gitlab-http/lib/httparty/response_patch.rb b/gems/gitlab-http/lib/httparty/response_patch.rb new file mode 100644 index 00000000000..3488ff034b4 --- /dev/null +++ b/gems/gitlab-http/lib/httparty/response_patch.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'httparty' + +HTTParty::Response.class_eval do + # Original method: https://github.com/jnunemaker/httparty/blob/v0.20.0/lib/httparty/response.rb#L83-L86 + # Related issue: https://github.com/jnunemaker/httparty/issues/568 + # + # We need to override this method because `Concurrent::Promise` calls `nil?` on the response when + # calling the `value` method. And the `value` calls `nil?`. + # https://github.com/ruby-concurrency/concurrent-ruby/blob/v1.2.2/lib/concurrent-ruby/concurrent/concern/dereferenceable.rb#L64 + def nil? + response.nil? || response.body.blank? + end +end diff --git a/gems/gitlab-http/lib/net_http/protocol_patch.rb b/gems/gitlab-http/lib/net_http/protocol_patch.rb new file mode 100644 index 00000000000..8231423e1a5 --- /dev/null +++ b/gems/gitlab-http/lib/net_http/protocol_patch.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +# Monkey patch Net::HTTP to fix missing URL decoding for username and password in proxy settings +# +# See proposed upstream fix https://github.com/ruby/net-http/pull/5 +# See Ruby-lang issue https://bugs.ruby-lang.org/issues/17542 +# See issue on GitLab https://gitlab.com/gitlab-org/gitlab/-/issues/289836 + +require 'net/http' + +# This file can be removed once Ruby 3.0 is no longer supported: +# https://gitlab.com/gitlab-org/gitlab/-/issues/396223 +return if Gem::Version.new(Net::HTTP::VERSION) >= Gem::Version.new('0.2.0') + +module Net + class HTTP < Protocol + def proxy_user + if environment_variable_is_multiuser_safe? && @proxy_from_env + user = proxy_uri&.user + CGI.unescape(user) unless user.nil? + else + @proxy_user + end + end + + def proxy_pass + if environment_variable_is_multiuser_safe? && @proxy_from_env + pass = proxy_uri&.password + CGI.unescape(pass) unless pass.nil? + else + @proxy_pass + end + end + + def environment_variable_is_multiuser_safe? + ENVIRONMENT_VARIABLE_IS_MULTIUSER_SAFE + end + end +end diff --git a/gems/gitlab-http/lib/net_http/response_patch.rb b/gems/gitlab-http/lib/net_http/response_patch.rb new file mode 100644 index 00000000000..e5477a31318 --- /dev/null +++ b/gems/gitlab-http/lib/net_http/response_patch.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Net + class HTTPResponse + # rubocop: disable Cop/LineBreakAfterGuardClauses + # rubocop: disable Cop/LineBreakAroundConditionalBlock + # rubocop: disable Layout/EmptyLineAfterGuardClause + # rubocop: disable Style/AndOr + # rubocop: disable Style/CharacterLiteral + # rubocop: disable Style/InfiniteLoop + + # Original method: + # https://github.com/ruby/ruby/blob/v2_7_5/lib/net/http/response.rb#L54-L69 + # + # Our changes: + # - Pass along the `start_time` to `Gitlab::HTTP_V2::BufferedIo`, so we can raise a timeout + # if reading the headers takes too long. + # - Limit the regexes to avoid ReDoS attacks. + def self.each_response_header(sock) + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + key = value = nil + while true + line = if sock.is_a?(Gitlab::HTTP_V2::BufferedIo) + sock.readuntil("\n", true, start_time) + else + sock.readuntil("\n", true) + end + line = line.sub(/\s{0,10}\z/, '') + break if line.empty? + if line[0] == ?\s or line[0] == ?\t and value + # rubocop:disable Gitlab/NoCodeCoverageComment + # :nocov: + value << ' ' unless value.empty? + value << line.strip + # :nocov: + # rubocop:enable Gitlab/NoCodeCoverageComment + else + yield key, value if key + key, value = line.strip.split(/\s{0,10}:\s{0,10}/, 2) + raise Net::HTTPBadResponse, 'wrong header line format' if value.nil? + end + end + yield key, value if key + end + # rubocop: enable Cop/LineBreakAfterGuardClauses + # rubocop: enable Cop/LineBreakAroundConditionalBlock + # rubocop: enable Layout/EmptyLineAfterGuardClause + # rubocop: enable Style/AndOr + # rubocop: enable Style/CharacterLiteral + # rubocop: enable Style/InfiniteLoop + end +end |