diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-17 21:09:44 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-03-17 21:09:44 +0300 |
commit | 2c156e3c7bbade01c36eee18327f1ced6eebea79 (patch) | |
tree | 115fa8dbf6bc05037378b380311d31acb805f54c /lib | |
parent | 8e129497b2565b8c595ef4f806d9a9595ca654e5 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/entities/discussion.rb | 2 | ||||
-rw-r--r-- | lib/api/entities/note_with_gitlab_employee_badge.rb | 10 | ||||
-rw-r--r-- | lib/api/entities/user_with_gitlab_employee_badge.rb | 9 | ||||
-rw-r--r-- | lib/gitlab/ci/config/entry/reports.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/ci/parsers.rb | 3 | ||||
-rw-r--r-- | lib/gitlab/ci/parsers/coverage/cobertura.rb | 64 | ||||
-rw-r--r-- | lib/gitlab/ci/reports/coverage_reports.rb | 43 | ||||
-rw-r--r-- | lib/gitlab/import_export/group/tree_saver.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/legacy_writer.rb | 73 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/streaming_serializer.rb | 82 | ||||
-rw-r--r-- | lib/gitlab/import_export/legacy_relation_tree_saver.rb (renamed from lib/gitlab/import_export/relation_tree_saver.rb) | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/legacy_tree_saver.rb | 68 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/tree_saver.rb | 46 | ||||
-rw-r--r-- | lib/tasks/gitlab/import_export/export.rake | 1 | ||||
-rw-r--r-- | lib/tasks/gitlab/import_export/import.rake | 1 | ||||
-rw-r--r-- | lib/tasks/sidekiq.rake | 2 |
16 files changed, 374 insertions, 37 deletions
diff --git a/lib/api/entities/discussion.rb b/lib/api/entities/discussion.rb index dd1dd40da23..0740de97897 100644 --- a/lib/api/entities/discussion.rb +++ b/lib/api/entities/discussion.rb @@ -5,7 +5,7 @@ module API class Discussion < Grape::Entity expose :id expose :individual_note?, as: :individual_note - expose :notes, using: Entities::Note + expose :notes, using: Entities::NoteWithGitlabEmployeeBadge end end end diff --git a/lib/api/entities/note_with_gitlab_employee_badge.rb b/lib/api/entities/note_with_gitlab_employee_badge.rb new file mode 100644 index 00000000000..2ea300ffeb6 --- /dev/null +++ b/lib/api/entities/note_with_gitlab_employee_badge.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module API + module Entities + class NoteWithGitlabEmployeeBadge < Note + expose :author, using: Entities::UserWithGitlabEmployeeBadge + expose :resolved_by, using: Entities::UserWithGitlabEmployeeBadge, if: ->(note, options) { note.resolvable? } + end + end +end diff --git a/lib/api/entities/user_with_gitlab_employee_badge.rb b/lib/api/entities/user_with_gitlab_employee_badge.rb new file mode 100644 index 00000000000..36b9f633132 --- /dev/null +++ b/lib/api/entities/user_with_gitlab_employee_badge.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module API + module Entities + class UserWithGitlabEmployeeBadge < UserBasic + expose :gitlab_employee?, as: :is_gitlab_employee, if: ->(user, options) { ::Feature.enabled?(:gitlab_employee_badge) && user.gitlab_employee? } + end + end +end diff --git a/lib/gitlab/ci/config/entry/reports.rb b/lib/gitlab/ci/config/entry/reports.rb index 994d3799004..40d37f3601a 100644 --- a/lib/gitlab/ci/config/entry/reports.rb +++ b/lib/gitlab/ci/config/entry/reports.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[junit codequality sast dependency_scanning container_scanning dast performance license_management license_scanning metrics lsif - dotenv].freeze + dotenv cobertura].freeze attributes ALLOWED_KEYS @@ -35,6 +35,7 @@ module Gitlab validates :metrics, array_of_strings_or_string: true validates :lsif, array_of_strings_or_string: true validates :dotenv, array_of_strings_or_string: true + validates :cobertura, array_of_strings_or_string: true end end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index c76cd5ff285..a44105d53c2 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -9,7 +9,8 @@ module Gitlab def self.parsers { - junit: ::Gitlab::Ci::Parsers::Test::Junit + junit: ::Gitlab::Ci::Parsers::Test::Junit, + cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura } end diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb new file mode 100644 index 00000000000..006d5097148 --- /dev/null +++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Coverage + class Cobertura + CoberturaParserError = Class.new(Gitlab::Ci::Parsers::ParserError) + + def parse!(xml_data, coverage_report) + root = Hash.from_xml(xml_data) + + parse_all(root, coverage_report) + rescue Nokogiri::XML::SyntaxError + raise CoberturaParserError, "XML parsing failed" + rescue + raise CoberturaParserError, "Cobertura parsing failed" + end + + private + + def parse_all(root, coverage_report) + return unless root.present? + + root.each do |key, value| + parse_node(key, value, coverage_report) + end + end + + def parse_node(key, value, coverage_report) + if key == 'class' + Array.wrap(value).each do |item| + parse_class(item, coverage_report) + end + elsif value.is_a?(Hash) + parse_all(value, coverage_report) + elsif value.is_a?(Array) + value.each do |item| + parse_all(item, coverage_report) + end + end + end + + def parse_class(file, coverage_report) + return unless file["filename"].present? && file["lines"].present? + + parsed_lines = parse_lines(file["lines"]) + + coverage_report.add_file(file["filename"], Hash[parsed_lines]) + end + + def parse_lines(lines) + line_array = Array.wrap(lines["line"]) + + line_array.map do |line| + # Using `Integer()` here to raise exception on invalid values + [Integer(line["number"]), Integer(line["hits"])] + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/coverage_reports.rb b/lib/gitlab/ci/reports/coverage_reports.rb new file mode 100644 index 00000000000..31afb636d2f --- /dev/null +++ b/lib/gitlab/ci/reports/coverage_reports.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + class CoverageReports + attr_reader :files + + def initialize + @files = {} + end + + def pick(keys) + coverage_files = files.select do |key| + keys.include?(key) + end + + { files: coverage_files } + end + + def add_file(name, line_coverage) + if files[name].present? + line_coverage.each { |line, hits| combine_lines(name, line, hits) } + + else + files[name] = line_coverage + end + end + + private + + def combine_lines(name, line, hits) + if files[name][line].present? + files[name][line] += hits + + else + files[name][line] = hits + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb index 48f6925884b..fd1eb329ad2 100644 --- a/lib/gitlab/import_export/group/tree_saver.rb +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -49,7 +49,7 @@ module Gitlab end def tree_saver - @tree_saver ||= RelationTreeSaver.new + @tree_saver ||= LegacyRelationTreeSaver.new end end end diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb new file mode 100644 index 00000000000..c935e360a65 --- /dev/null +++ b/lib/gitlab/import_export/json/legacy_writer.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module JSON + class LegacyWriter + include Gitlab::ImportExport::CommandLineUtil + + attr_reader :path + + def initialize(path) + @path = path + @last_array = nil + @keys = Set.new + + mkdir_p(File.dirname(@path)) + file.write('{}') + end + + def close + @file&.close + @file = nil + end + + def set(hash) + hash.each do |key, value| + write(key, value) + end + end + + def write(key, value) + raise ArgumentError, "key '#{key}' already written" if @keys.include?(key) + + # rewind by one byte, to overwrite '}' + file.pos = file.size - 1 + + file.write(',') if @keys.any? + file.write(key.to_json) + file.write(':') + file.write(value.to_json) + file.write('}') + + @keys.add(key) + @last_array = nil + @last_array_count = nil + end + + def append(key, value) + unless @last_array == key + write(key, []) + + @last_array = key + @last_array_count = 0 + end + + # rewind by two bytes, to overwrite ']}' + file.pos = file.size - 2 + + file.write(',') if @last_array_count > 0 + file.write(value.to_json) + file.write(']}') + @last_array_count += 1 + end + + private + + def file + @file ||= File.open(@path, "wb") + end + end + end + end +end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb new file mode 100644 index 00000000000..d053bf16166 --- /dev/null +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -0,0 +1,82 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module JSON + class StreamingSerializer + include Gitlab::ImportExport::CommandLineUtil + + BATCH_SIZE = 100 + + class Raw < String + def to_json(*_args) + to_s + end + end + + def initialize(exportable, relations_schema, json_writer) + @exportable = exportable + @relations_schema = relations_schema + @json_writer = json_writer + end + + def execute + serialize_root + + includes.each do |relation_definition| + serialize_relation(relation_definition) + end + end + + private + + attr_reader :json_writer, :relations_schema, :exportable + + def serialize_root + attributes = exportable.as_json( + relations_schema.merge(include: nil, preloads: nil)) + json_writer.set(attributes) + end + + def serialize_relation(definition) + raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash) + raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one? + + key, options = definition.first + + record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend + if record.is_a?(ActiveRecord::Relation) + serialize_many_relations(key, record, options) + else + serialize_single_relation(key, record, options) + end + end + + def serialize_many_relations(key, records, options) + key_preloads = preloads&.dig(key) + records = records.preload(key_preloads) if key_preloads + + records.find_each(batch_size: BATCH_SIZE) do |record| + json = Raw.new(record.to_json(options)) + + json_writer.append(key, json) + end + end + + def serialize_single_relation(key, record, options) + json = Raw.new(record.to_json(options)) + + json_writer.write(key, json) + end + + def includes + relations_schema[:include] + end + + def preloads + relations_schema[:preload] + end + end + end + end +end diff --git a/lib/gitlab/import_export/relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb index ed5392c13d0..fe3e64358e5 100644 --- a/lib/gitlab/import_export/relation_tree_saver.rb +++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb @@ -2,7 +2,7 @@ module Gitlab module ImportExport - class RelationTreeSaver + class LegacyRelationTreeSaver include Gitlab::ImportExport::CommandLineUtil def serialize(exportable, relations_tree) diff --git a/lib/gitlab/import_export/project/legacy_tree_saver.rb b/lib/gitlab/import_export/project/legacy_tree_saver.rb new file mode 100644 index 00000000000..2ed98f52c58 --- /dev/null +++ b/lib/gitlab/import_export/project/legacy_tree_saver.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class LegacyTreeSaver + attr_reader :full_path + + def initialize(project:, current_user:, shared:, params: {}) + @params = params + @project = project + @current_user = current_user + @shared = shared + @full_path = File.join(@shared.export_path, ImportExport.project_filename) + end + + def save + project_tree = tree_saver.serialize(@project, reader.project_tree) + fix_project_tree(project_tree) + tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) + + true + rescue => e + @shared.error(e) + false + end + + private + + # Aware that the resulting hash needs to be pure-hash and + # does not include any AR objects anymore, only objects that run `.to_json` + def fix_project_tree(project_tree) + if @params[:description].present? + project_tree['description'] = @params[:description] + end + + project_tree['project_members'] += group_members_array + end + + def reader + @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) + end + + def group_members_array + group_members.as_json(reader.group_members_tree).each do |group_member| + group_member['source_type'] = 'Project' # Make group members project members of the future import + end + end + + def group_members + return [] unless @current_user.can?(:admin_group, @project.group) + + # We need `.where.not(user_id: nil)` here otherwise when a group has an + # invitee, it would make the following query return 0 rows since a NULL + # user_id would be present in the subquery + # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values + non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id) + + GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) + end + + def tree_saver + @tree_saver ||= Gitlab::ImportExport::LegacyRelationTreeSaver.new + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 32b3b518ece..01000c9d6d9 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -15,52 +15,40 @@ module Gitlab end def save - project_tree = tree_saver.serialize(@project, reader.project_tree) - fix_project_tree(project_tree) - tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename) + json_writer = ImportExport::JSON::LegacyWriter.new(@full_path) + + serializer = ImportExport::JSON::StreamingSerializer.new(exportable, reader.project_tree, json_writer) + serializer.execute true rescue => e @shared.error(e) false + ensure + json_writer&.close end private - # Aware that the resulting hash needs to be pure-hash and - # does not include any AR objects anymore, only objects that run `.to_json` - def fix_project_tree(project_tree) - if @params[:description].present? - project_tree['description'] = @params[:description] - end - - project_tree['project_members'] += group_members_array - end - def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end - def group_members_array - group_members.as_json(reader.group_members_tree).each do |group_member| - group_member['source_type'] = 'Project' # Make group members project members of the future import - end + def exportable + @project.present(exportable_params) end - def group_members - return [] unless @current_user.can?(:admin_group, @project.group) - - # We need `.where.not(user_id: nil)` here otherwise when a group has an - # invitee, it would make the following query return 0 rows since a NULL - # user_id would be present in the subquery - # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values - non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id) - - GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids) + def exportable_params + params = { + presenter_class: presenter_class, + current_user: @current_user + } + params[:override_description] = @params[:description] if @params[:description].present? + params end - def tree_saver - @tree_saver ||= RelationTreeSaver.new + def presenter_class + Projects::ImportExport::ProjectExportPresenter end end end diff --git a/lib/tasks/gitlab/import_export/export.rake b/lib/tasks/gitlab/import_export/export.rake index ae54636fdd3..c9c212fbe4d 100644 --- a/lib/tasks/gitlab/import_export/export.rake +++ b/lib/tasks/gitlab/import_export/export.rake @@ -19,7 +19,6 @@ namespace :gitlab do if ENV['EXPORT_DEBUG'].present? ActiveRecord::Base.logger = logger - Gitlab::Metrics::Exporter::SidekiqExporter.instance.start logger.level = Logger::DEBUG else logger.level = Logger::INFO diff --git a/lib/tasks/gitlab/import_export/import.rake b/lib/tasks/gitlab/import_export/import.rake index 6e2d0e75da8..7e2162a7774 100644 --- a/lib/tasks/gitlab/import_export/import.rake +++ b/lib/tasks/gitlab/import_export/import.rake @@ -23,7 +23,6 @@ namespace :gitlab do if ENV['IMPORT_DEBUG'].present? ActiveRecord::Base.logger = logger - Gitlab::Metrics::Exporter::SidekiqExporter.instance.start logger.level = Logger::DEBUG else logger.level = Logger::INFO diff --git a/lib/tasks/sidekiq.rake b/lib/tasks/sidekiq.rake index e281ebd5d60..d74878835fd 100644 --- a/lib/tasks/sidekiq.rake +++ b/lib/tasks/sidekiq.rake @@ -33,6 +33,6 @@ namespace :sidekiq do task :launchd do deprecation_warning! - system(*%w(bin/background_jobs start_no_deamonize)) + system(*%w(bin/background_jobs start_silent)) end end |