diff options
Diffstat (limited to 'lib/gitlab')
32 files changed, 638 insertions, 137 deletions
diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index d575367d81a..fba80c7132e 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -14,28 +14,16 @@ module Gitlab # Public: Converts the provided Asciidoc markup into HTML. # # input - the source text in Asciidoc format - # context - a Hash with the template context: - # :commit - # :project - # :project_wiki - # :requested_path - # :ref - # asciidoc_opts - a Hash of options to pass to the Asciidoctor converter # - def self.render(input, context, asciidoc_opts = {}) - asciidoc_opts.reverse_merge!( - safe: :secure, - backend: :gitlab_html5, - attributes: [] - ) - asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) + def self.render(input) + asciidoc_opts = { safe: :secure, + backend: :gitlab_html5, + attributes: DEFAULT_ADOC_ATTRS } plantuml_setup html = ::Asciidoctor.convert(input, asciidoc_opts) - html = Banzai.post_process(html, context) - filter = Banzai::Filter::SanitizationFilter.new(html) html = filter.call.to_s diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index eee5601b0ed..ea918b23a63 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -108,7 +108,7 @@ module Gitlab token = Doorkeeper::AccessToken.by_token(password) if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) - Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) + Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) end end end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index b358f2efa4f..4fc9a075edc 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -15,18 +15,51 @@ module Gitlab end end + def self.load_in_batch_for_projects(projects) + cached_results_for_projects(projects).zip(projects).each do |result, project| + project.pipeline_status = new(project, result) + project.pipeline_status.load_status + end + end + + def self.cached_results_for_projects(projects) + result = Gitlab::Redis.with do |redis| + redis.multi do + projects.each do |project| + cache_key = cache_key_for_project(project) + redis.exists(cache_key) + redis.hmget(cache_key, :sha, :status, :ref) + end + end + end + + result.each_slice(2).map do |(cache_key_exists, (sha, status, ref))| + pipeline_info = { sha: sha, status: status, ref: ref } + { loaded_from_cache: cache_key_exists, pipeline_info: pipeline_info } + end + end + + def self.cache_key_for_project(project) + "projects/#{project.id}/pipeline_status" + end + def self.update_for_pipeline(pipeline) - new(pipeline.project, - sha: pipeline.sha, - status: pipeline.status, - ref: pipeline.ref).store_in_cache_if_needed + pipeline_info = { + sha: pipeline.sha, + status: pipeline.status, + ref: pipeline.ref + } + + new(pipeline.project, pipeline_info: pipeline_info). + store_in_cache_if_needed end - def initialize(project, sha: nil, status: nil, ref: nil) + def initialize(project, pipeline_info: {}, loaded_from_cache: nil) @project = project - @sha = sha - @ref = ref - @status = status + @sha = pipeline_info[:sha] + @ref = pipeline_info[:ref] + @status = pipeline_info[:status] + @loaded = loaded_from_cache end def has_status? @@ -85,6 +118,8 @@ module Gitlab end def has_cache? + return self.loaded unless self.loaded.nil? + Gitlab::Redis.with do |redis| redis.exists(cache_key) end @@ -95,7 +130,7 @@ module Gitlab end def cache_key - "projects/#{project.id}/build_status" + self.class.cache_key_for_project(project) end end end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index d76aa38f741..1ff34553f0a 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -41,7 +41,7 @@ module Gitlab type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push' # Hash to be passed as post_receive_data - data = { + { object_kind: type, event_name: type, before: oldrev, @@ -61,16 +61,15 @@ module Gitlab repository: project.hook_attrs.slice(:name, :url, :description, :homepage, :git_http_url, :git_ssh_url, :visibility_level) } - - data end # This method provide a sample data generated with # existing project and commits to test webhooks def build_sample(project, user) - commits = project.repository.commits(project.default_branch, limit: 3) ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" - build(project, user, commits.last.id, commits.first.id, ref, commits) + commits = project.repository.commits(project.default_branch.to_s, limit: 3) rescue [] + + build(project, user, commits.last&.id, commits.first&.id, ref, commits) end def checkout_sha(repository, newrev, ref) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 6dabbe0264c..298b1a1f4e6 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -498,6 +498,29 @@ module Gitlab columns(table).find { |column| column.name == name } end + + # This will replace the first occurance of a string in a column with + # the replacement + # On postgresql we can use `regexp_replace` for that. + # On mysql we find the location of the pattern, and overwrite it + # with the replacement + def replace_sql(column, pattern, replacement) + quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) + quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) + + if Database.mysql? + locate = Arel::Nodes::NamedFunction. + new('locate', [quoted_pattern, column]) + insert_in_place = Arel::Nodes::NamedFunction. + new('insert', [column, locate, pattern.size, quoted_replacement]) + + Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) + else + replace = Arel::Nodes::NamedFunction. + new("regexp_replace", [column, quoted_pattern, quoted_replacement]) + Arel::Nodes::SqlLiteral.new(replace.to_sql) + end + end end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb new file mode 100644 index 00000000000..89530082cd2 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb @@ -0,0 +1,35 @@ +# This module can be included in migrations to make it easier to rename paths +# of `Namespace` & `Project` models certain paths would become `reserved`. +# +# If the way things are stored on the filesystem related to namespaces and +# projects ever changes. Don't update this module, or anything nested in `V1`, +# since it needs to keep functioning for all migrations using it using the state +# that the data is in at the time. Instead, create a `V2` module that implements +# the new way of reserving paths. +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + def self.included(kls) + kls.include(MigrationHelpers) + end + + def rename_wildcard_paths(one_or_more_paths) + rename_child_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + RenameProjects.new(paths, self).rename_projects + end + + def rename_child_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + RenameNamespaces.new(paths, self).rename_namespaces(type: :child) + end + + def rename_root_paths(paths) + paths = Array(paths) + RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level) + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb new file mode 100644 index 00000000000..4fdcb682c2f --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -0,0 +1,76 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + module MigrationClasses + module Routable + def full_path + if route && route.path.present? + @full_path ||= route.path + else + update_route if persisted? + + build_full_path + end + end + + def build_full_path + if parent && path + parent.full_path + '/' + path + else + path + end + end + + def update_route + prepare_route + route.save + end + + def prepare_route + route || build_route(source: self) + route.path = build_full_path + @full_path = nil + end + end + + class Namespace < ActiveRecord::Base + include MigrationClasses::Routable + self.table_name = 'namespaces' + belongs_to :parent, + class_name: "#{MigrationClasses.name}::Namespace" + has_one :route, as: :source + has_many :children, + class_name: "#{MigrationClasses.name}::Namespace", + foreign_key: :parent_id + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Namespace' + end + end + + class Route < ActiveRecord::Base + self.table_name = 'routes' + belongs_to :source, polymorphic: true + end + + class Project < ActiveRecord::Base + include MigrationClasses::Routable + has_one :route, as: :source + self.table_name = 'projects' + + def repository_storage_path + Gitlab.config.repositories.storages[repository_storage]['path'] + end + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Project' + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb new file mode 100644 index 00000000000..de4e6e7c404 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -0,0 +1,131 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameBase + attr_reader :paths, :migration + + delegate :update_column_in_batches, + :replace_sql, + to: :migration + + def initialize(paths, migration) + @paths = paths + @migration = migration + end + + def path_patterns + @path_patterns ||= paths.map { |path| "%#{path}" } + end + + def rename_path_for_routable(routable) + old_path = routable.path + old_full_path = routable.full_path + # Only remove the last occurrence of the path name to get the parent namespace path + namespace_path = remove_last_occurrence(old_full_path, old_path) + new_path = rename_path(namespace_path, old_path) + new_full_path = join_routable_path(namespace_path, new_path) + + # skips callbacks & validations + routable.class.where(id: routable). + update_all(path: new_path) + + rename_routes(old_full_path, new_full_path) + + [old_full_path, new_full_path] + end + + def rename_routes(old_full_path, new_full_path) + replace_statement = replace_sql(Route.arel_table[:path], + old_full_path, + new_full_path) + + update_column_in_batches(:routes, :path, replace_statement) do |table, query| + query.where(MigrationClasses::Route.arel_table[:path].matches("#{old_full_path}%")) + end + end + + def rename_path(namespace_path, path_was) + counter = 0 + path = "#{path_was}#{counter}" + + while route_exists?(join_routable_path(namespace_path, path)) + counter += 1 + path = "#{path_was}#{counter}" + end + + path + end + + def remove_last_occurrence(string, pattern) + string.reverse.sub(pattern.reverse, "").reverse + end + + def join_routable_path(namespace_path, top_level) + if namespace_path.present? + File.join(namespace_path, top_level) + else + top_level + end + end + + def route_exists?(full_path) + MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any? + end + + def move_pages(old_path, new_path) + move_folders(pages_dir, old_path, new_path) + end + + def move_uploads(old_path, new_path) + return unless file_storage? + + move_folders(uploads_dir, old_path, new_path) + end + + def move_folders(directory, old_relative_path, new_relative_path) + old_path = File.join(directory, old_relative_path) + return unless File.directory?(old_path) + + new_path = File.join(directory, new_relative_path) + FileUtils.mv(old_path, new_path) + end + + def remove_cached_html_for_projects(project_ids) + update_column_in_batches(:projects, :description_html, nil) do |table, query| + query.where(table[:id].in(project_ids)) + end + + update_column_in_batches(:issues, :description_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + + update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| + query.where(table[:target_project_id].in(project_ids)) + end + + update_column_in_batches(:notes, :note_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + + update_column_in_batches(:milestones, :description_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + end + + def file_storage? + CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File + end + + def uploads_dir + File.join(CarrierWave.root, "uploads") + end + + def pages_dir + Settings.pages.path + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb new file mode 100644 index 00000000000..b9f4f3cff3c --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -0,0 +1,72 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameNamespaces < RenameBase + include Gitlab::ShellAdapter + + def rename_namespaces(type:) + namespaces_for_paths(type: type).each do |namespace| + rename_namespace(namespace) + end + end + + def namespaces_for_paths(type:) + namespaces = case type + when :child + MigrationClasses::Namespace.where.not(parent_id: nil) + when :top_level + MigrationClasses::Namespace.where(parent_id: nil) + end + with_paths = MigrationClasses::Route.arel_table[:path]. + matches_any(path_patterns) + namespaces.joins(:route).where(with_paths) + end + + def rename_namespace(namespace) + old_full_path, new_full_path = rename_path_for_routable(namespace) + + move_repositories(namespace, old_full_path, new_full_path) + move_uploads(old_full_path, new_full_path) + move_pages(old_full_path, new_full_path) + remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id)) + end + + def move_repositories(namespace, old_full_path, new_full_path) + repo_paths_for_namespace(namespace).each do |repository_storage_path| + # Ensure old directory exists before moving it + gitlab_shell.add_namespace(repository_storage_path, old_full_path) + + unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path) + message = "Exception moving path #{repository_storage_path} \ + from #{old_full_path} to #{new_full_path}" + Rails.logger.error message + end + end + end + + def repo_paths_for_namespace(namespace) + projects_for_namespace(namespace).distinct.select(:repository_storage). + map(&:repository_storage_path) + end + + def projects_for_namespace(namespace) + namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id]) + namespace_or_children = MigrationClasses::Project. + arel_table[:namespace_id]. + in(namespace_ids) + MigrationClasses::Project.where(namespace_or_children) + end + + def child_ids_for_parent(namespace, ids: []) + namespace.children.each do |child| + ids << child.id + child_ids_for_parent(child, ids: ids) if child.children.any? + end + ids + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb new file mode 100644 index 00000000000..448717eb744 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -0,0 +1,45 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameProjects < RenameBase + include Gitlab::ShellAdapter + + def rename_projects + projects_for_paths.each do |project| + rename_project(project) + end + + remove_cached_html_for_projects(projects_for_paths.map(&:id)) + end + + def rename_project(project) + old_full_path, new_full_path = rename_path_for_routable(project) + + move_repository(project, old_full_path, new_full_path) + move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") + move_uploads(old_full_path, new_full_path) + move_pages(old_full_path, new_full_path) + end + + def move_repository(project, old_path, new_path) + unless gitlab_shell.mv_repository(project.repository_storage_path, + old_path, + new_path) + Rails.logger.error "Error moving #{old_path} to #{new_path}" + end + end + + def projects_for_paths + return @projects_for_paths if @projects_for_paths + + with_paths = MigrationClasses::Route.arel_table[:path] + .matches_any(path_patterns) + + @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths) + end + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 329d12f13d1..0bd226ef050 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -15,6 +15,10 @@ module Gitlab super.tap { |_| store_highlight_cache } end + def real_size + @merge_request_diff.real_size + end + private # Extracted method to highlight in the same iteration to the diff_collection. diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index 4d04f867268..c7542a8fabc 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -82,7 +82,7 @@ module Gitlab file_diff, old_line, new_line = results - Position.new( + new_position = Position.new( old_path: file_diff.old_path, new_path: file_diff.new_path, head_sha: new_diff_refs.head_sha, @@ -91,6 +91,13 @@ module Gitlab old_line: old_line, new_line: new_line ) + + # If a position is found, but is not actually contained in the diff, for example + # because it was an unchanged line in the context of a change that was undone, + # we cannot return this as a successful trace. + return unless new_position.diff_line(repository) + + new_position end private diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 3f6ace0311a..0bba433d04b 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -16,6 +16,10 @@ module Gitlab def execute raise NotImplementedError end + + def metrics_params + { handler: self.class.name } + end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index b8ec9138c10..e7f91607e7e 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -1,4 +1,3 @@ - require 'gitlab/email/handler/base_handler' module Gitlab @@ -37,6 +36,10 @@ module Gitlab @project ||= Project.find_by_full_path(project_path) end + def metrics_params + super.merge(project: project) + end + private def create_issue diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 0e22f2189ee..31bb775c357 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -7,6 +7,8 @@ module Gitlab class CreateNoteHandler < BaseHandler include ReplyProcessing + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+\z/ end @@ -26,16 +28,16 @@ module Gitlab record_name: 'comment') end + def metrics_params + super.merge(project: project) + end + private def author sent_notification.recipient end - def project - sent_notification.project - end - def sent_notification @sent_notification ||= SentNotification.for(mail_key) end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 97d7a8d65ff..df70a063330 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -4,6 +4,8 @@ module Gitlab module Email module Handler class UnsubscribeHandler < BaseHandler + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ end @@ -17,6 +19,10 @@ module Gitlab noteable.unsubscribe(sent_notification.recipient) end + def metrics_params + super.merge(project: project) + end + private def sent_notification diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index ec0529b5a4b..c270c0ea9ff 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -1,4 +1,3 @@ - require_dependency 'gitlab/email/handler' # Inspired in great part by Discourse's Email::Receiver @@ -32,6 +31,8 @@ module Gitlab raise UnknownIncomingEmail unless handler + Gitlab::Metrics.add_event(:receive_email, handler.metrics_params) + handler.execute end @@ -69,6 +70,8 @@ module Gitlab # Handle emails from clients which append with commas, # example clients are Microsoft exchange and iOS app Gitlab::IncomingEmail.scan_fallback_references(references) + when nil + [] end end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index f6e4f279c06..aac210f19e8 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -2,31 +2,39 @@ module Gitlab module EtagCaching class Router Route = Struct.new(:regexp, :name) - - RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|') + # We enable an ETag for every request matching the regex. + # To match a regex the path needs to match the following: + # - Don't contain a reserved word (expect for the words used in the + # regex itself) + # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route + # - Ending in `issues/id`/rendered_title` for the `issue_title` route + USED_IN_ROUTES = %w[noteable issue notes issues rendered_title + commit pipelines merge_requests new].freeze + RESERVED_WORDS = DynamicPathValidator::WILDCARD_ROUTES - USED_IN_ROUTES + RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) ROUTES = [ Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z), 'issue_notes' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/issues/\d+/rendered_title\z), 'issue_title' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/commit/\S+/pipelines\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/commit/\S+/pipelines\.json\z), 'commit_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/merge_requests/new\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/new\.json\z), 'new_merge_request_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/merge_requests/\d+/pipelines\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/\d+/pipelines\.json\z), 'merge_request_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines\.json\z), 'project_pipelines' ) ].freeze diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 98fd4e78126..e8bb9e1f805 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -109,10 +109,6 @@ module Gitlab @binary.nil? ? super : @binary == true end - def empty? - !data || data == '' - end - def data encode! @data end diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb index e57d228e688..f918074cb14 100644 --- a/lib/gitlab/git/encoding_helper.rb +++ b/lib/gitlab/git/encoding_helper.rb @@ -40,7 +40,13 @@ module Gitlab def encode_utf8(message) detect = CharlockHolmes::EncodingDetector.detect(message) if detect - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end else clean(message) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d7dac9f6149..c3f0de76d01 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -122,13 +122,30 @@ module Gitlab # Returns the number of valid branches def branch_count - rugged.branches.count do |ref| - begin - ref.name && ref.target # ensures the branch is valid + Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled| + if is_enabled + gitaly_ref_client.count_branch_names + else + rugged.branches.count do |ref| + begin + ref.name && ref.target # ensures the branch is valid - true - rescue Rugged::ReferenceError - false + true + rescue Rugged::ReferenceError + false + end + end + end + end + end + + # Returns the number of valid tags + def tag_count + Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled| + if is_enabled + gitaly_ref_client.count_tag_names + else + rugged.tags.count end end end @@ -451,7 +468,7 @@ module Gitlab # Returns true is +from+ is direct ancestor to +to+, otherwise false def is_ancestor?(from, to) - Gitlab::GitalyClient::Commit.is_ancestor(self, from, to) + gitaly_commit_client.is_ancestor(from, to) end # Return an array of Diff objects that represent the diff @@ -494,7 +511,9 @@ module Gitlab # :contains is the commit contained by the refs from which to begin (SHA1 or name) # :max_count is the maximum number of commits to fetch # :skip is the number of commits to skip - # :order is the commits order and allowed value is :date(default) or :topo + # :order is the commits order and allowed value is :none (default), :date, or :topo + # commit ordering types are documented here: + # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) # def find_commits(options = {}) actual_options = options.dup @@ -522,11 +541,8 @@ module Gitlab end end - if actual_options[:order] == :topo - walker.sorting(Rugged::SORT_TOPO) - else - walker.sorting(Rugged::SORT_NONE) - end + sort_type = rugged_sort_type(actual_options[:order]) + walker.sorting(sort_type) commits = [] offset = actual_options[:skip] @@ -1273,6 +1289,22 @@ module Gitlab def gitaly_ref_client @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self) end + + def gitaly_commit_client + @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self) + end + + # Returns the `Rugged` sorting type constant for a given + # sort type key. Valid keys are `:none`, `:topo`, and `:date` + def rugged_sort_type(key) + @rugged_sort_types ||= { + none: Rugged::SORT_NONE, + topo: Rugged::SORT_TOPO, + date: Rugged::SORT_DATE + } + + @rugged_sort_types.fetch(key, Rugged::SORT_NONE) + end end end end diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index b7f39f3ef0b..27db1e19bc1 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -5,6 +5,23 @@ module Gitlab # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze + attr_accessor :stub + + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @stub = Gitaly::Commit::Stub.new(nil, nil, channel_override: repository.gitaly_channel) + end + + def is_ancestor(ancestor_id, child_id) + request = Gitaly::CommitIsAncestorRequest.new( + repository: @gitaly_repo, + ancestor_id: ancestor_id, + child_id: child_id + ) + + @stub.commit_is_ancestor(request).value + end + class << self def diff_from_parent(commit, options = {}) repository = commit.project.repository @@ -20,18 +37,6 @@ module Gitlab Gitlab::Git::DiffCollection.new(stub.commit_diff(request), options) end - - def is_ancestor(repository, ancestor_id, child_id) - gitaly_repo = repository.gitaly_repository - stub = Gitaly::Commit::Stub.new(nil, nil, channel_override: repository.gitaly_channel) - request = Gitaly::CommitIsAncestorRequest.new( - repository: gitaly_repo, - ancestor_id: ancestor_id, - child_id: child_id - ) - - stub.commit_is_ancestor(request).value - end end end end diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb index d3c0743db4e..2a5e8f73e55 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref.rb @@ -34,6 +34,14 @@ module Gitlab stub.find_ref_name(request).name end + def count_tag_names + tag_names.count + end + + def count_branch_names + branch_names.count + end + private def consume_refs_response(response, prefix:) diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index b02b9737493..5ca3e6a95ca 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -1,7 +1,23 @@ module Gitlab module GoogleCodeImport class Importer - attr_reader :project, :repo + attr_reader :project, :repo, :closed_statuses + + NICE_LABEL_COLOR_HASH = + { + 'Status: New' => '#428bca', + 'Status: Accepted' => '#5cb85c', + 'Status: Started' => '#8e44ad', + 'Priority: Critical' => '#ffcfcf', + 'Priority: High' => '#deffcf', + 'Priority: Medium' => '#fff5cc', + 'Priority: Low' => '#cfe9ff', + 'Type: Defect' => '#d9534f', + 'Type: Enhancement' => '#44ad8e', + 'Type: Task' => '#4b6dd0', + 'Type: Review' => '#8e44ad', + 'Type: Other' => '#7f8c8d' + }.freeze def initialize(project) @project = project @@ -161,45 +177,19 @@ module Gitlab end def nice_label_color(name) - case name - when /\AComponent:/ - "#fff39e" - when /\AOpSys:/ - "#e2e2e2" - when /\AMilestone:/ - "#fee3ff" - - when "Status: New" - "#428bca" - when "Status: Accepted" - "#5cb85c" - when "Status: Started" - "#8e44ad" - - when "Priority: Critical" - "#ffcfcf" - when "Priority: High" - "#deffcf" - when "Priority: Medium" - "#fff5cc" - when "Priority: Low" - "#cfe9ff" - - when "Type: Defect" - "#d9534f" - when "Type: Enhancement" - "#44ad8e" - when "Type: Task" - "#4b6dd0" - when "Type: Review" - "#8e44ad" - when "Type: Other" - "#7f8c8d" - when *@closed_statuses.map { |s| nice_status_name(s) } - "#cfcfcf" - else - "#e2e2e2" - end + NICE_LABEL_COLOR_HASH[name] || + case name + when /\AComponent:/ + '#fff39e' + when /\AOpSys:/ + '#e2e2e2' + when /\AMilestone:/ + '#fee3ff' + when *closed_statuses.map { |s| nice_status_name(s) } + '#cfcfcf' + else + '#e2e2e2' + end end def nice_label_name(name) diff --git a/lib/gitlab/issuable_sorter.rb b/lib/gitlab/issuable_sorter.rb new file mode 100644 index 00000000000..d392214867a --- /dev/null +++ b/lib/gitlab/issuable_sorter.rb @@ -0,0 +1,29 @@ +module Gitlab + module IssuableSorter + class << self + def sort(project, issuables, &sort_key) + grouped_items = issuables.group_by do |issuable| + if issuable.project.id == project.id + :project_ref + elsif issuable.project.namespace.id == project.namespace.id + :namespace_ref + else + :full_ref + end + end + + natural_sort_issuables(grouped_items[:project_ref], project) + + natural_sort_issuables(grouped_items[:namespace_ref], project) + + natural_sort_issuables(grouped_items[:full_ref], project) + end + + private + + def natural_sort_issuables(issuables, project) + VersionSorter.sort(issuables || []) do |issuable| + issuable.to_reference(project) + end + end + end + end +end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 857e0abf710..c6dfa4ad9bd 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -138,6 +138,11 @@ module Gitlab @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' end + # Allow access from other metrics related middlewares + def self.current_transaction + Transaction.current + end + # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? @@ -149,10 +154,5 @@ module Gitlab new(udp: { host: host, port: port }) end end - - # Allow access from other metrics related middlewares - def self.current_transaction - Transaction.current - end end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 6e42d8941fb..afd24b4dcc5 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -148,7 +148,7 @@ module Gitlab def build_new_user user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) - Users::BuildService.new(nil, user_params).execute + Users::BuildService.new(nil, user_params).execute(skip_authorization: true) end def user_attributes diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index e67acf28c94..c2adc9aa10b 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -4,19 +4,11 @@ module Gitlab # Public: Converts the provided markup into HTML. # # input - the source text in a markup format - # context - a Hash with the template context: - # :commit - # :project - # :project_wiki - # :requested_path - # :ref # - def self.render(file_name, input, context) + def self.render(file_name, input) html = GitHub::Markup.render(file_name, input). force_encoding(input.encoding) - html = Banzai.post_process(html, context) - filter = Banzai::Filter::SanitizationFilter.new(html) html = filter.call.to_s diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 08b061d5e31..b7fef5dd068 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -22,6 +22,10 @@ module Gitlab @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze end + def full_namespace_regex + @full_namespace_regex ||= %r{\A#{FULL_NAMESPACE_REGEX_STR}\z} + end + def namespace_route_regex @namespace_route_regex ||= /#{NAMESPACE_REGEX_STR}/.freeze end diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb index d5d3e045a42..20b054b0bd8 100644 --- a/lib/gitlab/template/dockerfile_template.rb +++ b/lib/gitlab/template/dockerfile_template.rb @@ -8,7 +8,7 @@ module Gitlab class << self def extension - 'Dockerfile' + '.Dockerfile' end def categories @@ -18,7 +18,7 @@ module Gitlab end def base_dir - Rails.root.join('vendor/dockerfile') + Rails.root.join('vendor/Dockerfile') end def finder(project = nil) diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 54728e5ff0e..e46ff313654 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -44,9 +44,7 @@ module Gitlab if ProtectedBranch.protected?(project, ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) - has_access = project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push) - - has_access || !project.repository.branch_exists?(ref) && can_merge_to_branch?(ref) + project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push) else user.can?(:push_code, project) end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e6e40f6945d..c551f939df1 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -168,7 +168,7 @@ module Gitlab end def secret_path - Rails.root.join('.gitlab_workhorse_secret') + Gitlab.config.workhorse.secret_file end def set_key_and_notify(key, value, expire: nil, overwrite: true) |