diff options
Diffstat (limited to 'lib/gitlab')
-rw-r--r-- | lib/gitlab/auth.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/gfm/uploads_rewriter.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/hotlinking_detector.rb | 52 | ||||
-rw-r--r-- | lib/gitlab/import_export/attribute_cleaner.rb | 9 | ||||
-rw-r--r-- | lib/gitlab/import_export/group/tree_restorer.rb | 58 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/legacy_reader.rb | 56 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/legacy_writer.rb | 63 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/streaming_serializer.rb | 25 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/tree_restorer.rb | 29 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/tree_saver.rb | 15 | ||||
-rw-r--r-- | lib/gitlab/import_export/relation_tree_restorer.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/regex.rb | 8 |
12 files changed, 254 insertions, 85 deletions
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index c16c2ce96de..7f7bdda953f 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -171,6 +171,8 @@ module Gitlab if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) + return unless user.can?(:log_in) + Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) end end @@ -182,7 +184,7 @@ module Gitlab token = PersonalAccessTokensFinder.new(state: 'active').find_by_token(password) - if token && valid_scoped_token?(token, all_available_scopes) + if token && valid_scoped_token?(token, all_available_scopes) && token.user.can?(:log_in) Gitlab::Auth::Result.new(token.user, nil, :personal_access_token, abilities_for_scopes(token.scopes)) end end @@ -260,6 +262,8 @@ module Gitlab return unless build.project.builds_enabled? if build.user + return unless build.user.can?(:log_in) + # If user is assigned to build, use restricted credentials of user Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities) else diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 6b52d6e88e5..23af0a9bb18 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -22,6 +22,8 @@ module Gitlab return @text unless needs_rewrite? @text.gsub(@pattern) do |markdown| + Gitlab::Utils.check_path_traversal!($~[:file]) + file = find_file(@source_project, $~[:secret], $~[:file]) break markdown unless file.try(:exists?) diff --git a/lib/gitlab/hotlinking_detector.rb b/lib/gitlab/hotlinking_detector.rb new file mode 100644 index 00000000000..44901297870 --- /dev/null +++ b/lib/gitlab/hotlinking_detector.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + class HotlinkingDetector + IMAGE_FORMATS = %w(image/jpeg image/apng image/png image/webp image/svg+xml image/*).freeze + MEDIA_FORMATS = %w(video/webm video/ogg video/* application/ogg audio/webm audio/ogg audio/wav audio/*).freeze + CSS_FORMATS = %w(text/css).freeze + INVALID_FORMATS = (IMAGE_FORMATS + MEDIA_FORMATS + CSS_FORMATS).freeze + INVALID_FETCH_MODES = %w(cors no-cors websocket).freeze + + class << self + def intercept_hotlinking?(request) + request_accepts = parse_request_accepts(request) + + return false unless Feature.enabled?(:repository_archive_hotlinking_interception, default_enabled: true) + + # Block attempts to embed as JS + return true if sec_fetch_invalid?(request) + + # If no Accept header was set, skip the rest + return false if request_accepts.empty? + + # Workaround for IE8 weirdness + return false if IMAGE_FORMATS.include?(request_accepts.first) && request_accepts.include?("application/x-ms-application") + + # Block all other media requests if the first format is a media type + return true if INVALID_FORMATS.include?(request_accepts.first) + + false + end + + private + + def sec_fetch_invalid?(request) + fetch_mode = request.headers["Sec-Fetch-Mode"] + + return if fetch_mode.blank? + return true if INVALID_FETCH_MODES.include?(fetch_mode) + end + + def parse_request_accepts(request) + # Rails will already have parsed the Accept header + return request.accepts if request.respond_to?(:accepts) + + # Grape doesn't parse it, so we can use the Rails system for this + return Mime::Type.parse(request.headers["Accept"]) if request.respond_to?(:headers) && request.headers["Accept"].present? + + [] + end + end + end +end diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb index 3bfc059dcd3..018cb36fc58 100644 --- a/lib/gitlab/import_export/attribute_cleaner.rb +++ b/lib/gitlab/import_export/attribute_cleaner.rb @@ -11,7 +11,14 @@ module Gitlab 'discussion_id', 'custom_attributes' ].freeze - PROHIBITED_REFERENCES = Regexp.union(/\Acached_markdown_version\Z/, /_id\Z/, /_ids\Z/, /_html\Z/, /attributes/).freeze + PROHIBITED_REFERENCES = Regexp.union( + /\Acached_markdown_version\Z/, + /_id\Z/, + /_ids\Z/, + /_html\Z/, + /attributes/, + /\Aremote_\w+_(url|urls|request_header)\Z/ # carrierwave automatically creates these attribute methods for uploads + ).freeze def self.clean(*args) new(*args).clean diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index 247e39a68b9..f6ebd83bfaa 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -4,12 +4,13 @@ module Gitlab module ImportExport module Group class TreeRestorer + include Gitlab::Utils::StrongMemoize + attr_reader :user attr_reader :shared attr_reader :group def initialize(user:, shared:, group:, group_hash:) - @path = File.join(shared.export_path, 'group.json') @user = user @shared = shared @group = group @@ -17,17 +18,14 @@ module Gitlab end def restore - @relation_reader ||= - if @group_hash.present? - ImportExport::JSON::LegacyReader::User.new(@group_hash, reader.group_relation_names) - else - ImportExport::JSON::LegacyReader::File.new(@path, reader.group_relation_names) - end + @group_attributes = relation_reader.consume_attributes(nil) + @group_members = relation_reader.consume_relation(nil, 'members') + + # We need to remove `name` and `path` as we did consume it in previous pass + @group_attributes.delete('name') + @group_attributes.delete('path') - @group_members = @relation_reader.consume_relation('members') - @children = @relation_reader.consume_attribute('children') - @relation_reader.consume_attribute('name') - @relation_reader.consume_attribute('path') + @children = @group_attributes.delete('children') if members_mapper.map && restorer.restore @children&.each do |group_hash| @@ -53,16 +51,32 @@ module Gitlab private + def relation_reader + strong_memoize(:relation_reader) do + if @group_hash.present? + ImportExport::JSON::LegacyReader::Hash.new( + @group_hash, + relation_names: reader.group_relation_names) + else + ImportExport::JSON::LegacyReader::File.new( + File.join(shared.export_path, 'group.json'), + relation_names: reader.group_relation_names) + end + end + end + def restorer @relation_tree_restorer ||= RelationTreeRestorer.new( - user: @user, - shared: @shared, - importable: @group, - relation_reader: @relation_reader, - members_mapper: members_mapper, - object_builder: object_builder, - relation_factory: relation_factory, - reader: reader + user: @user, + shared: @shared, + relation_reader: relation_reader, + members_mapper: members_mapper, + object_builder: object_builder, + relation_factory: relation_factory, + reader: reader, + importable: @group, + importable_attributes: @group_attributes, + importable_path: nil ) end @@ -88,7 +102,11 @@ module Gitlab end def members_mapper - @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group) + @members_mapper ||= Gitlab::ImportExport::MembersMapper.new( + exported_members: @group_members, + user: @user, + importable: @group + ) end def relation_factory diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb index 477e41ae3eb..57579fe9def 100644 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -5,19 +5,25 @@ module Gitlab module JSON class LegacyReader class File < LegacyReader - def initialize(path, relation_names) + include Gitlab::Utils::StrongMemoize + + def initialize(path, relation_names:, allowed_path: nil) @path = path - super(relation_names) + super( + relation_names: relation_names, + allowed_path: allowed_path) end - def valid? + def exist? ::File.exist?(@path) end - private + protected def tree_hash - @tree_hash ||= read_hash + strong_memoize(:tree_hash) do + read_hash + end end def read_hash @@ -28,13 +34,15 @@ module Gitlab end end - class User < LegacyReader - def initialize(tree_hash, relation_names) + class Hash < LegacyReader + def initialize(tree_hash, relation_names:, allowed_path: nil) @tree_hash = tree_hash - super(relation_names) + super( + relation_names: relation_names, + allowed_path: allowed_path) end - def valid? + def exist? @tree_hash.present? end @@ -43,11 +51,16 @@ module Gitlab attr_reader :tree_hash end - def initialize(relation_names) + def initialize(relation_names:, allowed_path:) @relation_names = relation_names.map(&:to_s) + + # This is legacy reader, to be used in transition + # period before `.ndjson`, + # we strong validate what is being readed + @allowed_path = allowed_path end - def valid? + def exist? raise NotImplementedError end @@ -55,15 +68,22 @@ module Gitlab true end - def root_attributes(excluded_attributes = []) - attributes.except(*excluded_attributes.map(&:to_s)) + def consume_attributes(importable_path) + unless importable_path == @allowed_path + raise ArgumentError, "Invalid #{importable_path} passed to `consume_attributes`. Use #{@allowed_path} instead." + end + + attributes end - def consume_relation(key) + def consume_relation(importable_path, key) + unless importable_path == @allowed_path + raise ArgumentError, "Invalid #{importable_name} passed to `consume_relation`. Use #{@allowed_path} instead." + end + value = relations.delete(key) return value unless block_given? - return if value.nil? if value.is_a?(Array) @@ -75,17 +95,13 @@ module Gitlab end end - def consume_attribute(key) - attributes.delete(key) - end - def sort_ci_pipelines_by_id relations['ci_pipelines']&.sort_by! { |hash| hash['id'] } end private - attr_reader :relation_names + attr_reader :relation_names, :allowed_path def tree_hash raise NotImplementedError diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb index c935e360a65..7be21410d26 100644 --- a/lib/gitlab/import_export/json/legacy_writer.rb +++ b/lib/gitlab/import_export/json/legacy_writer.rb @@ -8,11 +8,15 @@ module Gitlab attr_reader :path - def initialize(path) + def initialize(path, allowed_path:) @path = path - @last_array = nil @keys = Set.new + # This is legacy writer, to be used in transition + # period before `.ndjson`, + # we strong validate what is being written + @allowed_path = allowed_path + mkdir_p(File.dirname(@path)) file.write('{}') end @@ -22,12 +26,44 @@ module Gitlab @file = nil end - def set(hash) + def write_attributes(exportable_path, hash) + unless exportable_path == @allowed_path + raise ArgumentError, "Invalid #{exportable_path}" + end + hash.each do |key, value| write(key, value) end end + def write_relation(exportable_path, key, value) + unless exportable_path == @allowed_path + raise ArgumentError, "Invalid #{exportable_path}" + end + + write(key, value) + end + + def write_relation_array(exportable_path, key, items) + unless exportable_path == @allowed_path + raise ArgumentError, "Invalid #{exportable_path}" + end + + write(key, []) + + # rewind by two bytes, to overwrite ']}' + file.pos = file.size - 2 + + items.each_with_index do |item, idx| + file.write(',') if idx > 0 + file.write(item.to_json) + end + + file.write(']}') + end + + private + def write(key, value) raise ArgumentError, "key '#{key}' already written" if @keys.include?(key) @@ -41,29 +77,8 @@ module Gitlab file.write('}') @keys.add(key) - @last_array = nil - @last_array_count = nil - end - - def append(key, value) - unless @last_array == key - write(key, []) - - @last_array = key - @last_array_count = 0 - end - - # rewind by two bytes, to overwrite ']}' - file.pos = file.size - 2 - - file.write(',') if @last_array_count > 0 - file.write(value.to_json) - file.write(']}') - @last_array_count += 1 end - private - def file @file ||= File.open(@path, "wb") end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index d053bf16166..04ca598ff5d 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -14,8 +14,9 @@ module Gitlab end end - def initialize(exportable, relations_schema, json_writer) + def initialize(exportable, relations_schema, json_writer, exportable_path:) @exportable = exportable + @exportable_path = exportable_path @relations_schema = relations_schema @json_writer = json_writer end @@ -35,7 +36,7 @@ module Gitlab def serialize_root attributes = exportable.as_json( relations_schema.merge(include: nil, preloads: nil)) - json_writer.set(attributes) + json_writer.write_attributes(@exportable_path, attributes) end def serialize_relation(definition) @@ -47,16 +48,28 @@ module Gitlab record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend if record.is_a?(ActiveRecord::Relation) serialize_many_relations(key, record, options) + elsif record.respond_to?(:each) # this is to support `project_members` that return an Array + serialize_many_each(key, record, options) else serialize_single_relation(key, record, options) end end def serialize_many_relations(key, records, options) - key_preloads = preloads&.dig(key) - records = records.preload(key_preloads) if key_preloads + enumerator = Enumerator.new do |items| + key_preloads = preloads&.dig(key) + records = records.preload(key_preloads) if key_preloads - records.find_each(batch_size: BATCH_SIZE) do |record| + records.find_each(batch_size: BATCH_SIZE) do |record| + items << Raw.new(record.to_json(options)) + end + end + + json_writer.write_relation_array(@exportable_path, key, enumerator) + end + + def serialize_many_each(key, records, options) + records.each do |record| json = Raw.new(record.to_json(options)) json_writer.append(key, json) @@ -66,7 +79,7 @@ module Gitlab def serialize_single_relation(key, record, options) json = Raw.new(record.to_json(options)) - json_writer.write(key, json) + json_writer.write_relation(@exportable_path, key, json) end def includes diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index f8d25e14c02..99e57d9decd 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -4,6 +4,8 @@ module Gitlab module ImportExport module Project class TreeRestorer + include Gitlab::Utils::StrongMemoize + attr_reader :user attr_reader :shared attr_reader :project @@ -15,9 +17,8 @@ module Gitlab end def restore - @relation_reader = ImportExport::JSON::LegacyReader::File.new(File.join(shared.export_path, 'project.json'), reader.project_relation_names) - - @project_members = @relation_reader.consume_relation('project_members') + @project_attributes = relation_reader.consume_attributes(importable_path) + @project_members = relation_reader.consume_relation(importable_path, 'project_members') if relation_tree_restorer.restore import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do @@ -35,16 +36,28 @@ module Gitlab private + def relation_reader + strong_memoize(:relation_reader) do + ImportExport::JSON::LegacyReader::File.new( + File.join(shared.export_path, 'project.json'), + relation_names: reader.project_relation_names, + allowed_path: importable_path + ) + end + end + def relation_tree_restorer @relation_tree_restorer ||= RelationTreeRestorer.new( user: @user, shared: @shared, - importable: @project, - relation_reader: @relation_reader, + relation_reader: relation_reader, object_builder: object_builder, members_mapper: members_mapper, relation_factory: relation_factory, - reader: reader + reader: reader, + importable: @project, + importable_attributes: @project_attributes, + importable_path: importable_path ) end @@ -69,6 +82,10 @@ module Gitlab def import_failure_service @import_failure_service ||= ImportFailureService.new(@project) end + + def importable_path + "project" + end end end end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 01000c9d6d9..988776fe600 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -15,10 +15,17 @@ module Gitlab end def save - json_writer = ImportExport::JSON::LegacyWriter.new(@full_path) - - serializer = ImportExport::JSON::StreamingSerializer.new(exportable, reader.project_tree, json_writer) - serializer.execute + json_writer = ImportExport::JSON::LegacyWriter.new( + @full_path, + allowed_path: "project" + ) + + ImportExport::JSON::StreamingSerializer.new( + exportable, + reader.project_tree, + json_writer, + exportable_path: "project" + ).execute true rescue => e diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 1157e18c7f9..7f25222ba41 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -11,7 +11,15 @@ module Gitlab attr_reader :importable attr_reader :relation_reader - def initialize(user:, shared:, importable:, relation_reader:, members_mapper:, object_builder:, relation_factory:, reader:) + def initialize( # rubocop:disable Metrics/ParameterLists + user:, shared:, relation_reader:, + members_mapper:, object_builder:, + relation_factory:, + reader:, + importable:, + importable_attributes:, + importable_path: + ) @user = user @shared = shared @importable = importable @@ -20,6 +28,8 @@ module Gitlab @object_builder = object_builder @relation_factory = relation_factory @reader = reader + @importable_attributes = importable_attributes + @importable_path = importable_path end def restore @@ -57,7 +67,7 @@ module Gitlab end def process_relation!(relation_key, relation_definition) - @relation_reader.consume_relation(relation_key) do |data_hash, relation_index| + @relation_reader.consume_relation(@importable_path, relation_key) do |data_hash, relation_index| process_relation_item!(relation_key, relation_definition, relation_index, data_hash) end end @@ -94,7 +104,7 @@ module Gitlab end def update_params! - params = @relation_reader.root_attributes(relations.keys) + params = @importable_attributes.except(*relations.keys.map(&:to_s)) params = params.merge(present_override_params) # Cleaning all imported and overridden params diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 38281fb1c91..66503621851 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -16,6 +16,14 @@ module Gitlab "It must start with letter, digit, emoji or '_'." end + def group_name_regex + project_name_regex + end + + def group_name_regex_message + project_name_regex_message + end + ## # Docker Distribution Registry repository / tag name rules # |