diff options
Diffstat (limited to 'lib')
75 files changed, 1732 insertions, 332 deletions
diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 64ab6f01eb5..6d6ccefe877 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -14,7 +14,6 @@ module API class User < UserBasic expose :created_at - expose :admin?, as: :is_admin expose :bio, :location, :skype, :linkedin, :twitter, :website_url, :organization end @@ -41,8 +40,9 @@ module API expose :external end - class UserWithPrivateToken < UserPublic + class UserWithPrivateDetails < UserPublic expose :private_token + expose :admin?, as: :is_admin end class Email < Grape::Entity diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index ddff3c8c1e8..86bf567fe69 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -102,7 +102,7 @@ module API end def authenticate! - unauthorized! unless current_user && can?(current_user, :access_api) + unauthorized! unless current_user && can?(initial_current_user, :access_api) end def authenticate_non_get! diff --git a/lib/api/internal.rb b/lib/api/internal.rb index 5b48ee8665f..ebed26dd178 100644 --- a/lib/api/internal.rb +++ b/lib/api/internal.rb @@ -140,7 +140,7 @@ module API begin Gitlab::GitalyClient::Notifications.new(project.repository).post_receive rescue GRPC::Unavailable => e - render_api_error(e, 500) + render_api_error!(e, 500) end end end diff --git a/lib/api/issues.rb b/lib/api/issues.rb index 244725bb292..522f0f3be92 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -219,6 +219,21 @@ module API authorize!(:destroy_issue, issue) issue.destroy end + + desc 'List merge requests closing issue' do + success Entities::MergeRequestBasic + end + params do + requires :issue_iid, type: Integer, desc: 'The internal ID of a project issue' + end + get ':id/issues/:issue_iid/closed_by' do + issue = find_project_issue(params[:issue_iid]) + + merge_request_ids = MergeRequestsClosingIssues.where(issue_id: issue).select(:merge_request_id) + merge_requests = MergeRequestsFinder.new(current_user, project_id: user_project.id).execute.where(id: merge_request_ids) + + present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project + end end end end diff --git a/lib/api/merge_requests.rb b/lib/api/merge_requests.rb index cb7aec47cf0..710deba5ae3 100644 --- a/lib/api/merge_requests.rb +++ b/lib/api/merge_requests.rb @@ -20,6 +20,8 @@ module API error!(errors[:validate_fork], 422) elsif errors[:validate_branches].any? conflict!(errors[:validate_branches]) + elsif errors[:base].any? + error!(errors[:base], 422) end render_api_error!(errors, 400) @@ -33,6 +35,17 @@ module API end end + def find_merge_requests(args = {}) + args = params.merge(args) + + args[:milestone_title] = args.delete(:milestone) + args[:label_name] = args.delete(:labels) + + merge_requests = MergeRequestsFinder.new(current_user, args).execute.inc_notes_with_associations + + merge_requests.reorder(args[:order_by] => args[:sort]) + end + params :optional_params_ce do optional :description, type: String, desc: 'The description of the merge request' optional :assignee_id, type: Integer, desc: 'The ID of a user to assign the merge request' @@ -57,23 +70,15 @@ module API optional :sort, type: String, values: %w[asc desc], default: 'desc', desc: 'Return merge requests sorted in `asc` or `desc` order.' optional :iids, type: Array[Integer], desc: 'The IID array of merge requests' + optional :milestone, type: String, desc: 'Return merge requests for a specific milestone' + optional :labels, type: String, desc: 'Comma-separated list of label names' use :pagination end get ":id/merge_requests" do authorize! :read_merge_request, user_project - merge_requests = user_project.merge_requests.inc_notes_with_associations - merge_requests = filter_by_iid(merge_requests, params[:iids]) if params[:iids].present? - - merge_requests = - case params[:state] - when 'opened' then merge_requests.opened - when 'closed' then merge_requests.closed - when 'merged' then merge_requests.merged - else merge_requests - end + merge_requests = find_merge_requests(project_id: user_project.id) - merge_requests = merge_requests.reorder(params[:order_by] => params[:sort]) present paginate(merge_requests), with: Entities::MergeRequestBasic, current_user: current_user, project: user_project end @@ -197,14 +202,15 @@ module API end put ':id/merge_requests/:merge_request_iid/merge' do merge_request = find_project_merge_request(params[:merge_request_iid]) + merge_when_pipeline_succeeds = to_boolean(params[:merge_when_pipeline_succeeds]) # Merge request can not be merged # because user dont have permissions to push into target branch unauthorized! unless merge_request.can_be_merged_by?(current_user) - not_allowed! unless merge_request.mergeable_state? + not_allowed! unless merge_request.mergeable_state?(skip_ci_check: merge_when_pipeline_succeeds) - render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable? + render_api_error!('Branch cannot be merged', 406) unless merge_request.mergeable?(skip_ci_check: merge_when_pipeline_succeeds) if params[:sha] && merge_request.diff_head_sha != params[:sha] render_api_error!("SHA does not match HEAD of source branch: #{merge_request.diff_head_sha}", 409) @@ -215,7 +221,7 @@ module API should_remove_source_branch: params[:should_remove_source_branch] } - if params[:merge_when_pipeline_succeeds] && merge_request.head_pipeline && merge_request.head_pipeline.active? + if merge_when_pipeline_succeeds && merge_request.head_pipeline && merge_request.head_pipeline.active? ::MergeRequests::MergeWhenPipelineSucceedsService .new(merge_request.target_project, current_user, merge_params) .execute(merge_request) diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 50842370947..db4b31b55bc 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -11,7 +11,7 @@ module API optional :issues_enabled, type: Boolean, desc: 'Flag indication if the issue tracker is enabled' optional :merge_requests_enabled, type: Boolean, desc: 'Flag indication if merge requests are enabled' optional :wiki_enabled, type: Boolean, desc: 'Flag indication if the wiki is enabled' - optional :builds_enabled, type: Boolean, desc: 'Flag indication if builds are enabled' + optional :jobs_enabled, type: Boolean, desc: 'Flag indication if jobs are enabled' optional :snippets_enabled, type: Boolean, desc: 'Flag indication if snippets are enabled' optional :shared_runners_enabled, type: Boolean, desc: 'Flag indication if shared runners are enabled for that project' optional :container_registry_enabled, type: Boolean, desc: 'Flag indication if the container registry is enabled for that project' @@ -103,6 +103,7 @@ module API end post do attrs = declared_params(include_missing: false) + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) project = ::Projects::CreateService.new(current_user, attrs).execute if project.saved? @@ -205,7 +206,7 @@ module API # CE at_least_one_of_ce = [ - :builds_enabled, + :jobs_enabled, :container_registry_enabled, :default_branch, :description, @@ -236,6 +237,8 @@ module API authorize! :rename_project, user_project if attrs[:name].present? authorize! :change_visibility_level, user_project if attrs[:visibility].present? + attrs[:builds_enabled] = attrs.delete(:jobs_enabled) if attrs.has_key?(:jobs_enabled) + result = ::Projects::UpdateService.new(user_project, current_user, attrs).execute if result[:status] == :success diff --git a/lib/api/session.rb b/lib/api/session.rb index 002ffd1d154..016415c3023 100644 --- a/lib/api/session.rb +++ b/lib/api/session.rb @@ -1,7 +1,7 @@ module API class Session < Grape::API desc 'Login to get token' do - success Entities::UserWithPrivateToken + success Entities::UserWithPrivateDetails end params do optional :login, type: String, desc: 'The username' @@ -14,7 +14,7 @@ module API return unauthorized! unless user return render_api_error!('401 Unauthorized. You have 2FA enabled. Please use a personal access token to access the API', 401) if user.two_factor_enabled? - present user, with: Entities::UserWithPrivateToken + present user, with: Entities::UserWithPrivateDetails end end end diff --git a/lib/api/users.rb b/lib/api/users.rb index 46f221f68fe..40acaebf670 100644 --- a/lib/api/users.rb +++ b/lib/api/users.rb @@ -433,7 +433,7 @@ module API success Entities::UserPublic end get do - present current_user, with: sudo? ? Entities::UserWithPrivateToken : Entities::UserPublic + present current_user, with: sudo? ? Entities::UserWithPrivateDetails : Entities::UserPublic end desc "Get the currently authenticated user's SSH keys" do diff --git a/lib/api/v3/merge_requests.rb b/lib/api/v3/merge_requests.rb index 3077240e650..1616142a619 100644 --- a/lib/api/v3/merge_requests.rb +++ b/lib/api/v3/merge_requests.rb @@ -23,6 +23,8 @@ module API error!(errors[:validate_fork], 422) elsif errors[:validate_branches].any? conflict!(errors[:validate_branches]) + elsif errors[:base].any? + error!(errors[:base], 422) end render_api_error!(errors, 400) diff --git a/lib/backup/database.rb b/lib/backup/database.rb index 4016ac76348..d97e5d98229 100644 --- a/lib/backup/database.rb +++ b/lib/backup/database.rb @@ -80,16 +80,32 @@ module Backup 'port' => '--port', 'socket' => '--socket', 'username' => '--user', - 'encoding' => '--default-character-set' + 'encoding' => '--default-character-set', + # SSL + 'sslkey' => '--ssl-key', + 'sslcert' => '--ssl-cert', + 'sslca' => '--ssl-ca', + 'sslcapath' => '--ssl-capath', + 'sslcipher' => '--ssl-cipher' } args.map { |opt, arg| "#{arg}=#{config[opt]}" if config[opt] }.compact end def pg_env - ENV['PGUSER'] = config["username"] if config["username"] - ENV['PGHOST'] = config["host"] if config["host"] - ENV['PGPORT'] = config["port"].to_s if config["port"] - ENV['PGPASSWORD'] = config["password"].to_s if config["password"] + args = { + 'username' => 'PGUSER', + 'host' => 'PGHOST', + 'port' => 'PGPORT', + 'password' => 'PGPASSWORD', + # SSL + 'sslmode' => 'PGSSLMODE', + 'sslkey' => 'PGSSLKEY', + 'sslcert' => 'PGSSLCERT', + 'sslrootcert' => 'PGSSLROOTCERT', + 'sslcrl' => 'PGSSLCRL', + 'sslcompression' => 'PGSSLCOMPRESSION' + } + args.each { |opt, arg| ENV[arg] = config[opt].to_s if config[opt] } end def report_success(success) diff --git a/lib/backup/manager.rb b/lib/backup/manager.rb index 7b4476fa4db..330cd963626 100644 --- a/lib/backup/manager.rb +++ b/lib/backup/manager.rb @@ -15,11 +15,10 @@ module Backup s[:gitlab_version] = Gitlab::VERSION s[:tar_version] = tar_version s[:skipped] = ENV["SKIP"] - tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d')}#{FILE_NAME_SUFFIX}" + tar_file = "#{s[:backup_created_at].strftime('%s_%Y_%m_%d_')}#{s[:gitlab_version]}#{FILE_NAME_SUFFIX}" - Dir.chdir(Gitlab.config.backup.path) do - File.open("#{Gitlab.config.backup.path}/backup_information.yml", - "w+") do |file| + Dir.chdir(backup_path) do + File.open("#{backup_path}/backup_information.yml", "w+") do |file| file << s.to_yaml.gsub(/^---\n/, '') end @@ -64,9 +63,9 @@ module Backup $progress.print "Deleting tmp directories ... " backup_contents.each do |dir| - next unless File.exist?(File.join(Gitlab.config.backup.path, dir)) + next unless File.exist?(File.join(backup_path, dir)) - if FileUtils.rm_rf(File.join(Gitlab.config.backup.path, dir)) + if FileUtils.rm_rf(File.join(backup_path, dir)) $progress.puts "done".color(:green) else puts "deleting tmp directory '#{dir}' failed".color(:red) @@ -83,8 +82,8 @@ module Backup if keep_time > 0 removed = 0 - Dir.chdir(Gitlab.config.backup.path) do - Dir.glob("*#{FILE_NAME_SUFFIX}").each do |file| + Dir.chdir(backup_path) do + backup_file_list.each do |file| next unless file =~ /(\d+)(?:_\d{4}_\d{2}_\d{2})?_gitlab_backup\.tar/ timestamp = $1.to_i @@ -107,18 +106,14 @@ module Backup end def unpack - Dir.chdir(Gitlab.config.backup.path) + Dir.chdir(backup_path) # check for existing backups in the backup dir - file_list = Dir.glob("*#{FILE_NAME_SUFFIX}") - - if file_list.count == 0 - $progress.puts "No backups found in #{Gitlab.config.backup.path}" + if backup_file_list.empty? + $progress.puts "No backups found in #{backup_path}" $progress.puts "Please make sure that file name ends with #{FILE_NAME_SUFFIX}" exit 1 - end - - if file_list.count > 1 && ENV["BACKUP"].nil? + elsif backup_file_list.many? && ENV["BACKUP"].nil? $progress.puts 'Found more than one backup, please specify which one you want to restore:' $progress.puts 'rake gitlab:backup:restore BACKUP=timestamp_of_backup' exit 1 @@ -127,7 +122,7 @@ module Backup tar_file = if ENV['BACKUP'].present? "#{ENV['BACKUP']}#{FILE_NAME_SUFFIX}" else - file_list.first + backup_file_list.first end unless File.exist?(tar_file) @@ -169,6 +164,14 @@ module Backup private + def backup_path + Gitlab.config.backup.path + end + + def backup_file_list + @backup_file_list ||= Dir.glob("*#{FILE_NAME_SUFFIX}") + end + def connect_to_remote_directory(connection_settings) connection = ::Fog::Storage.new(connection_settings) diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index d6138816e70..6255a611dbe 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -53,7 +53,10 @@ module Banzai # Build a regexp that matches all valid :emoji: names. def self.emoji_pattern - @emoji_pattern ||= /:(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}):/ + @emoji_pattern ||= + /(?<=[^[:alnum:]:]|\n|^) + :(#{Gitlab::Emoji.emojis_names.map { |name| Regexp.escape(name) }.join('|')}): + (?=[^[:alnum:]:]|$)/x end # Build a regexp that matches all valid unicode emojis names. diff --git a/lib/banzai/filter/issuable_state_filter.rb b/lib/banzai/filter/issuable_state_filter.rb index 1a9d03beb51..327ea9449a1 100644 --- a/lib/banzai/filter/issuable_state_filter.rb +++ b/lib/banzai/filter/issuable_state_filter.rb @@ -15,8 +15,8 @@ module Banzai issuables = extractor.extract([doc]) issuables.each do |node, issuable| - if VISIBLE_STATES.include?(issuable.state) && node.children.present? - node.add_child(Nokogiri::XML::Text.new(" [#{issuable.state}]", doc)) + if VISIBLE_STATES.include?(issuable.state) && node.inner_html == issuable.reference_link_text(project) + node.content += " (#{issuable.state})" end end diff --git a/lib/banzai/filter/plantuml_filter.rb b/lib/banzai/filter/plantuml_filter.rb index b2537117558..5325819d828 100644 --- a/lib/banzai/filter/plantuml_filter.rb +++ b/lib/banzai/filter/plantuml_filter.rb @@ -7,14 +7,14 @@ module Banzai # class PlantumlFilter < HTML::Pipeline::Filter def call - return doc unless doc.at('pre.plantuml') && settings.plantuml_enabled + return doc unless doc.at('pre > code[lang="plantuml"]') && settings.plantuml_enabled plantuml_setup - doc.css('pre.plantuml').each do |el| + doc.css('pre > code[lang="plantuml"]').each do |node| img_tag = Nokogiri::HTML::DocumentFragment.parse( - Asciidoctor::PlantUml::Processor.plantuml_content(el.content, {})) - el.replace img_tag + Asciidoctor::PlantUml::Processor.plantuml_content(node.content, {})) + node.parent.replace(img_tag) end doc diff --git a/lib/banzai/issuable_extractor.rb b/lib/banzai/issuable_extractor.rb index c5ce360e172..cbabf9156de 100644 --- a/lib/banzai/issuable_extractor.rb +++ b/lib/banzai/issuable_extractor.rb @@ -28,9 +28,13 @@ module Banzai issue_parser = Banzai::ReferenceParser::IssueParser.new(project, user) merge_request_parser = Banzai::ReferenceParser::MergeRequestParser.new(project, user) - issue_parser.issues_for_nodes(nodes).merge( + issuables_for_nodes = issue_parser.issues_for_nodes(nodes).merge( merge_request_parser.merge_requests_for_nodes(nodes) ) + + # The project for the issue/MR might be pending for deletion! + # Filter them out because we don't care about them. + issuables_for_nodes.select { |node, issuable| issuable.project } end end end diff --git a/lib/banzai/renderer.rb b/lib/banzai/renderer.rb index 74663556cbb..c7801cb5baf 100644 --- a/lib/banzai/renderer.rb +++ b/lib/banzai/renderer.rb @@ -1,7 +1,5 @@ module Banzai module Renderer - module_function - # Convert a Markdown String into an HTML-safe String of HTML # # Note that while the returned HTML will have been sanitized of dangerous @@ -16,7 +14,7 @@ module Banzai # context - Hash of context options passed to our HTML Pipeline # # Returns an HTML-safe String - def render(text, context = {}) + def self.render(text, context = {}) cache_key = context.delete(:cache_key) cache_key = full_cache_key(cache_key, context[:pipeline]) @@ -35,24 +33,16 @@ module Banzai # of HTML. This method is analogous to calling render(object.field), but it # can cache the rendered HTML in the object, rather than Redis. # - # The context to use is learned from the passed-in object by calling - # #banzai_render_context(field), and cannot be changed. Use #render, passing - # it the field text, if a custom rendering is needed. The generated context - # is returned along with the HTML. - def render_field(object, field) - html_field = object.markdown_cache_field_for(field) - - html = object.__send__(html_field) - return html if html.present? - - html = cacheless_render_field(object, field) - update_object(object, html_field, html) unless object.new_record? || object.destroyed? + # The context to use is managed by the object and cannot be changed. + # Use #render, passing it the field text, if a custom rendering is needed. + def self.render_field(object, field) + object.refresh_markdown_cache!(do_update: update_object?(object)) unless object.cached_html_up_to_date?(field) - html + object.cached_html_for(field) end # Same as +render_field+, but without consulting or updating the cache field - def cacheless_render_field(object, field, options = {}) + def self.cacheless_render_field(object, field, options = {}) text = object.__send__(field) context = object.banzai_render_context(field).merge(options) @@ -82,7 +72,7 @@ module Banzai # texts_and_contexts # => [{ text: '### Hello', # context: { cache_key: [note, :note] } }] - def cache_collection_render(texts_and_contexts) + def self.cache_collection_render(texts_and_contexts) items_collection = texts_and_contexts.each_with_index do |item, index| context = item[:context] cache_key = full_cache_multi_key(context.delete(:cache_key), context[:pipeline]) @@ -111,7 +101,7 @@ module Banzai items_collection.map { |item| item[:rendered] } end - def render_result(text, context = {}) + def self.render_result(text, context = {}) text = Pipeline[:pre_process].to_html(text, context) if text Pipeline[context[:pipeline]].call(text, context) @@ -130,7 +120,7 @@ module Banzai # :user - User object # # Returns an HTML-safe String - def post_process(html, context) + def self.post_process(html, context) context = Pipeline[context[:pipeline]].transform_context(context) pipeline = Pipeline[:post_process] @@ -141,7 +131,7 @@ module Banzai end.html_safe end - def cacheless_render(text, context = {}) + def self.cacheless_render(text, context = {}) Gitlab::Metrics.measure(:banzai_cacheless_render) do result = render_result(text, context) @@ -154,7 +144,7 @@ module Banzai end end - def full_cache_key(cache_key, pipeline_name) + def self.full_cache_key(cache_key, pipeline_name) return unless cache_key ["banzai", *cache_key, pipeline_name || :full] end @@ -162,13 +152,14 @@ module Banzai # To map Rails.cache.read_multi results we need to know the Rails.cache.expanded_key. # Other option will be to generate stringified keys on our side and don't delegate to Rails.cache.expanded_key # method. - def full_cache_multi_key(cache_key, pipeline_name) + def self.full_cache_multi_key(cache_key, pipeline_name) return unless cache_key Rails.cache.send(:expanded_key, full_cache_key(cache_key, pipeline_name)) end - def update_object(object, html_field, html) - object.update_column(html_field, html) + # GitLab EE needs to disable updates on GET requests in Geo + def self.update_object?(object) + true end end end diff --git a/lib/constraints/group_url_constrainer.rb b/lib/constraints/group_url_constrainer.rb index bae4db1ca4d..1501f64d537 100644 --- a/lib/constraints/group_url_constrainer.rb +++ b/lib/constraints/group_url_constrainer.rb @@ -2,16 +2,8 @@ class GroupUrlConstrainer def matches?(request) id = request.params[:id] - return false unless valid?(id) + return false unless DynamicPathValidator.valid?(id) Group.find_by_full_path(id).present? end - - private - - def valid?(id) - id.split('/').all? do |namespace| - NamespaceValidator.valid?(namespace) - end - end end diff --git a/lib/constraints/project_url_constrainer.rb b/lib/constraints/project_url_constrainer.rb index a10b4657d7d..d0ce2caffff 100644 --- a/lib/constraints/project_url_constrainer.rb +++ b/lib/constraints/project_url_constrainer.rb @@ -4,9 +4,7 @@ class ProjectUrlConstrainer project_path = request.params[:project_id] || request.params[:id] full_path = namespace_path + '/' + project_path - unless ProjectPathValidator.valid?(project_path) - return false - end + return false unless DynamicPathValidator.valid?(full_path) Project.find_by_full_path(full_path).present? end diff --git a/lib/github/client.rb b/lib/github/client.rb new file mode 100644 index 00000000000..e65d908d232 --- /dev/null +++ b/lib/github/client.rb @@ -0,0 +1,23 @@ +module Github + class Client + attr_reader :connection, :rate_limit + + def initialize(options) + @connection = Faraday.new(url: options.fetch(:url)) do |faraday| + faraday.options.open_timeout = options.fetch(:timeout, 60) + faraday.options.timeout = options.fetch(:timeout, 60) + faraday.authorization 'token', options.fetch(:token) + faraday.adapter :net_http + end + + @rate_limit = RateLimit.new(connection) + end + + def get(url, query = {}) + exceed, reset_in = rate_limit.get + sleep reset_in if exceed + + Github::Response.new(connection.get(url, query)) + end + end +end diff --git a/lib/github/collection.rb b/lib/github/collection.rb new file mode 100644 index 00000000000..014b2038c4b --- /dev/null +++ b/lib/github/collection.rb @@ -0,0 +1,29 @@ +module Github + class Collection + attr_reader :options + + def initialize(options) + @options = options + end + + def fetch(url, query = {}) + return [] if url.blank? + + Enumerator.new do |yielder| + loop do + response = client.get(url, query) + response.body.each { |item| yielder << item } + + raise StopIteration unless response.rels.key?(:next) + url = response.rels[:next] + end + end.lazy + end + + private + + def client + @client ||= Github::Client.new(options) + end + end +end diff --git a/lib/github/error.rb b/lib/github/error.rb new file mode 100644 index 00000000000..66d7afaa787 --- /dev/null +++ b/lib/github/error.rb @@ -0,0 +1,3 @@ +module Github + RepositoryFetchError = Class.new(StandardError) +end diff --git a/lib/github/import.rb b/lib/github/import.rb new file mode 100644 index 00000000000..d49761fd6c6 --- /dev/null +++ b/lib/github/import.rb @@ -0,0 +1,409 @@ +require_relative 'error' +module Github + class Import + include Gitlab::ShellAdapter + + class MergeRequest < ::MergeRequest + self.table_name = 'merge_requests' + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class Issue < ::Issue + self.table_name = 'issues' + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class Note < ::Note + self.table_name = 'notes' + + self.reset_callbacks :save + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + class LegacyDiffNote < ::LegacyDiffNote + self.table_name = 'notes' + + self.reset_callbacks :commit + self.reset_callbacks :update + self.reset_callbacks :validate + end + + attr_reader :project, :repository, :repo, :options, :errors, :cached, :verbose + + def initialize(project, options) + @project = project + @repository = project.repository + @repo = project.import_source + @options = options + @verbose = options.fetch(:verbose, false) + @cached = Hash.new { |hash, key| hash[key] = Hash.new } + @errors = [] + end + + # rubocop: disable Rails/Output + def execute + puts 'Fetching repository...'.color(:aqua) if verbose + fetch_repository + puts 'Fetching labels...'.color(:aqua) if verbose + fetch_labels + puts 'Fetching milestones...'.color(:aqua) if verbose + fetch_milestones + puts 'Fetching pull requests...'.color(:aqua) if verbose + fetch_pull_requests + puts 'Fetching issues...'.color(:aqua) if verbose + fetch_issues + puts 'Cloning wiki repository...'.color(:aqua) if verbose + fetch_wiki_repository + puts 'Expiring repository cache...'.color(:aqua) if verbose + expire_repository_cache + + true + rescue Github::RepositoryFetchError + false + ensure + keep_track_of_errors + end + + private + + def fetch_repository + begin + project.create_repository unless project.repository.exists? + project.repository.add_remote('github', "https://{options.fetch(:token)}@github.com/#{repo}.git") + project.repository.set_remote_as_mirror('github') + project.repository.fetch_remote('github', forced: true) + rescue Gitlab::Shell::Error => e + error(:project, "https://github.com/#{repo}.git", e.message) + raise Github::RepositoryFetchError + end + end + + def fetch_wiki_repository + wiki_url = "https://{options.fetch(:token)}@github.com/#{repo}.wiki.git" + wiki_path = "#{project.path_with_namespace}.wiki" + + unless project.wiki.repository_exists? + gitlab_shell.import_repository(project.repository_storage_path, wiki_path, wiki_url) + end + rescue Gitlab::Shell::Error => e + # GitHub error message when the wiki repo has not been created, + # this means that repo has wiki enabled, but have no pages. So, + # we can skip the import. + if e.message !~ /repository not exported/ + errors(:wiki, wiki_url, e.message) + end + end + + def fetch_labels + url = "/repos/#{repo}/labels" + + while url + response = Github::Client.new(options).get(url) + + response.body.each do |raw| + begin + representation = Github::Representation::Label.new(raw) + + label = project.labels.find_or_create_by!(title: representation.title) do |label| + label.color = representation.color + end + + cached[:label_ids][label.title] = label.id + rescue => e + error(:label, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def fetch_milestones + url = "/repos/#{repo}/milestones" + + while url + response = Github::Client.new(options).get(url, state: :all) + + response.body.each do |raw| + begin + milestone = Github::Representation::Milestone.new(raw) + next if project.milestones.where(iid: milestone.iid).exists? + + project.milestones.create!( + iid: milestone.iid, + title: milestone.title, + description: milestone.description, + due_date: milestone.due_date, + state: milestone.state, + created_at: milestone.created_at, + updated_at: milestone.updated_at + ) + rescue => e + error(:milestone, milestone.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def fetch_pull_requests + url = "/repos/#{repo}/pulls" + + while url + response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) + + response.body.each do |raw| + pull_request = Github::Representation::PullRequest.new(raw, options.merge(project: project)) + merge_request = MergeRequest.find_or_initialize_by(iid: pull_request.iid, source_project_id: project.id) + next unless merge_request.new_record? && pull_request.valid? + + begin + restore_branches(pull_request) + + author_id = user_id(pull_request.author, project.creator_id) + description = format_description(pull_request.description, pull_request.author) + + merge_request.attributes = { + iid: pull_request.iid, + title: pull_request.title, + description: description, + source_project: pull_request.source_project, + source_branch: pull_request.source_branch_name, + source_branch_sha: pull_request.source_branch_sha, + target_project: pull_request.target_project, + target_branch: pull_request.target_branch_name, + target_branch_sha: pull_request.target_branch_sha, + state: pull_request.state, + milestone_id: milestone_id(pull_request.milestone), + author_id: author_id, + assignee_id: user_id(pull_request.assignee), + created_at: pull_request.created_at, + updated_at: pull_request.updated_at + } + + merge_request.save!(validate: false) + merge_request.merge_request_diffs.create + + # Fetch review comments + review_comments_url = "/repos/#{repo}/pulls/#{pull_request.iid}/comments" + fetch_comments(merge_request, :review_comment, review_comments_url, LegacyDiffNote) + + # Fetch comments + comments_url = "/repos/#{repo}/issues/#{pull_request.iid}/comments" + fetch_comments(merge_request, :comment, comments_url) + rescue => e + error(:pull_request, pull_request.url, e.message) + ensure + clean_up_restored_branches(pull_request) + end + end + + url = response.rels[:next] + end + end + + def fetch_issues + url = "/repos/#{repo}/issues" + + while url + response = Github::Client.new(options).get(url, state: :all, sort: :created, direction: :asc) + + response.body.each do |raw| + representation = Github::Representation::Issue.new(raw, options) + + begin + # Every pull request is an issue, but not every issue + # is a pull request. For this reason, "shared" actions + # for both features, like manipulating assignees, labels + # and milestones, are provided within the Issues API. + if representation.pull_request? + next unless representation.has_labels? + + merge_request = MergeRequest.find_by!(target_project_id: project.id, iid: representation.iid) + merge_request.update_attribute(:label_ids, label_ids(representation.labels)) + else + next if Issue.where(iid: representation.iid, project_id: project.id).exists? + + author_id = user_id(representation.author, project.creator_id) + issue = Issue.new + issue.iid = representation.iid + issue.project_id = project.id + issue.title = representation.title + issue.description = format_description(representation.description, representation.author) + issue.state = representation.state + issue.label_ids = label_ids(representation.labels) + issue.milestone_id = milestone_id(representation.milestone) + issue.author_id = author_id + issue.assignee_id = user_id(representation.assignee) + issue.created_at = representation.created_at + issue.updated_at = representation.updated_at + issue.save!(validate: false) + + # Fetch comments + if representation.has_comments? + comments_url = "/repos/#{repo}/issues/#{issue.iid}/comments" + fetch_comments(issue, :comment, comments_url) + end + end + rescue => e + error(:issue, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def fetch_comments(noteable, type, url, klass = Note) + while url + comments = Github::Client.new(options).get(url) + + ActiveRecord::Base.no_touching do + comments.body.each do |raw| + begin + representation = Github::Representation::Comment.new(raw, options) + author_id = user_id(representation.author, project.creator_id) + + note = klass.new + note.project_id = project.id + note.noteable = noteable + note.note = format_description(representation.note, representation.author) + note.commit_id = representation.commit_id + note.line_code = representation.line_code + note.author_id = author_id + note.created_at = representation.created_at + note.updated_at = representation.updated_at + note.save!(validate: false) + rescue => e + error(type, representation.url, e.message) + end + end + end + + url = comments.rels[:next] + end + end + + def fetch_releases + url = "/repos/#{repo}/releases" + + while url + response = Github::Client.new(options).get(url) + + response.body.each do |raw| + representation = Github::Representation::Release.new(raw) + next unless representation.valid? + + release = ::Release.find_or_initialize_by(project_id: project.id, tag: representation.tag) + next unless relese.new_record? + + begin + release.description = representation.description + release.created_at = representation.created_at + release.updated_at = representation.updated_at + release.save!(validate: false) + rescue => e + error(:release, representation.url, e.message) + end + end + + url = response.rels[:next] + end + end + + def restore_branches(pull_request) + restore_source_branch(pull_request) unless pull_request.source_branch_exists? + restore_target_branch(pull_request) unless pull_request.target_branch_exists? + end + + def restore_source_branch(pull_request) + repository.create_branch(pull_request.source_branch_name, pull_request.source_branch_sha) + end + + def restore_target_branch(pull_request) + repository.create_branch(pull_request.target_branch_name, pull_request.target_branch_sha) + end + + def remove_branch(name) + repository.delete_branch(name) + rescue Rugged::ReferenceError + errors << { type: :branch, url: nil, error: "Could not clean up restored branch: #{name}" } + end + + def clean_up_restored_branches(pull_request) + return if pull_request.opened? + + remove_branch(pull_request.source_branch_name) unless pull_request.source_branch_exists? + remove_branch(pull_request.target_branch_name) unless pull_request.target_branch_exists? + end + + def label_ids(labels) + labels.map { |attrs| cached[:label_ids][attrs.fetch('name')] }.compact + end + + def milestone_id(milestone) + return unless milestone.present? + + project.milestones.select(:id).find_by(iid: milestone.iid)&.id + end + + def user_id(user, fallback_id = nil) + return unless user.present? + return cached[:user_ids][user.id] if cached[:user_ids].key?(user.id) + + gitlab_user_id = user_id_by_external_uid(user.id) || user_id_by_email(user.email) + + cached[:gitlab_user_ids][user.id] = gitlab_user_id.present? + cached[:user_ids][user.id] = gitlab_user_id || fallback_id + end + + def user_id_by_email(email) + return nil unless email + + ::User.find_by_any_email(email)&.id + end + + def user_id_by_external_uid(id) + return nil unless id + + ::User.select(:id) + .joins(:identities) + .merge(::Identity.where(provider: :github, extern_uid: id)) + .first&.id + end + + def format_description(body, author) + return body if cached[:gitlab_user_ids][author.id] + + "*Created by: #{author.username}*\n\n#{body}" + end + + def expire_repository_cache + repository.expire_content_cache + end + + def keep_track_of_errors + return unless errors.any? + + project.update_column(:import_error, { + message: 'The remote data could not be fully imported.', + errors: errors + }.to_json) + end + + def error(type, url, message) + errors << { type: type, url: Gitlab::UrlSanitizer.sanitize(url), error: message } + end + end +end diff --git a/lib/github/rate_limit.rb b/lib/github/rate_limit.rb new file mode 100644 index 00000000000..884693d093c --- /dev/null +++ b/lib/github/rate_limit.rb @@ -0,0 +1,27 @@ +module Github + class RateLimit + SAFE_REMAINING_REQUESTS = 100 + SAFE_RESET_TIME = 500 + RATE_LIMIT_URL = '/rate_limit'.freeze + + attr_reader :connection + + def initialize(connection) + @connection = connection + end + + def get + response = connection.get(RATE_LIMIT_URL) + + # GitHub Rate Limit API returns 404 when the rate limit is disabled + return false unless response.status != 404 + + body = Oj.load(response.body, class_cache: false, mode: :compat) + remaining = body.dig('rate', 'remaining').to_i + reset_in = body.dig('rate', 'reset').to_i + exceed = remaining <= SAFE_REMAINING_REQUESTS + + [exceed, reset_in] + end + end +end diff --git a/lib/github/repositories.rb b/lib/github/repositories.rb new file mode 100644 index 00000000000..c1c9448f305 --- /dev/null +++ b/lib/github/repositories.rb @@ -0,0 +1,19 @@ +module Github + class Repositories + attr_reader :options + + def initialize(options) + @options = options + end + + def fetch + Collection.new(options).fetch(repos_url) + end + + private + + def repos_url + '/user/repos' + end + end +end diff --git a/lib/github/representation/base.rb b/lib/github/representation/base.rb new file mode 100644 index 00000000000..f26bdbdd546 --- /dev/null +++ b/lib/github/representation/base.rb @@ -0,0 +1,30 @@ +module Github + module Representation + class Base + def initialize(raw, options = {}) + @raw = raw + @options = options + end + + def id + raw['id'] + end + + def url + raw['url'] + end + + def created_at + raw['created_at'] + end + + def updated_at + raw['updated_at'] + end + + private + + attr_reader :raw, :options + end + end +end diff --git a/lib/github/representation/branch.rb b/lib/github/representation/branch.rb new file mode 100644 index 00000000000..d1dac6944f0 --- /dev/null +++ b/lib/github/representation/branch.rb @@ -0,0 +1,51 @@ +module Github + module Representation + class Branch < Representation::Base + attr_reader :repository + + def user + raw.dig('user', 'login') || 'unknown' + end + + def repo + return @repo if defined?(@repo) + + @repo = Github::Representation::Repo.new(raw['repo']) if raw['repo'].present? + end + + def ref + raw['ref'] + end + + def sha + raw['sha'] + end + + def short_sha + Commit.truncate_sha(sha) + end + + def exists? + branch_exists? && commit_exists? + end + + def valid? + sha.present? && ref.present? + end + + private + + def branch_exists? + repository.branch_exists?(ref) + end + + def commit_exists? + repository.branch_names_contains(sha).include?(ref) + end + + def repository + @repository ||= options.fetch(:repository) + end + end + end +end diff --git a/lib/github/representation/comment.rb b/lib/github/representation/comment.rb new file mode 100644 index 00000000000..1b5be91461b --- /dev/null +++ b/lib/github/representation/comment.rb @@ -0,0 +1,42 @@ +module Github + module Representation + class Comment < Representation::Base + def note + raw['body'] || '' + end + + def author + @author ||= Github::Representation::User.new(raw['user'], options) + end + + def commit_id + raw['commit_id'] + end + + def line_code + return unless on_diff? + + parsed_lines = Gitlab::Diff::Parser.new.parse(diff_hunk.lines) + generate_line_code(parsed_lines.to_a.last) + end + + private + + def generate_line_code(line) + Gitlab::Diff::LineCode.generate(file_path, line.new_pos, line.old_pos) + end + + def on_diff? + diff_hunk.present? + end + + def diff_hunk + raw['diff_hunk'] + end + + def file_path + raw['path'] + end + end + end +end diff --git a/lib/github/representation/issuable.rb b/lib/github/representation/issuable.rb new file mode 100644 index 00000000000..9713b82615d --- /dev/null +++ b/lib/github/representation/issuable.rb @@ -0,0 +1,37 @@ +module Github + module Representation + class Issuable < Representation::Base + def iid + raw['number'] + end + + def title + raw['title'] + end + + def description + raw['body'] || '' + end + + def milestone + return unless raw['milestone'].present? + + @milestone ||= Github::Representation::Milestone.new(raw['milestone']) + end + + def author + @author ||= Github::Representation::User.new(raw['user'], options) + end + + def assignee + return unless assigned? + + @assignee ||= Github::Representation::User.new(raw['assignee'], options) + end + + def assigned? + raw['assignee'].present? + end + end + end +end diff --git a/lib/github/representation/issue.rb b/lib/github/representation/issue.rb new file mode 100644 index 00000000000..df3540a6e6c --- /dev/null +++ b/lib/github/representation/issue.rb @@ -0,0 +1,25 @@ +module Github + module Representation + class Issue < Representation::Issuable + def labels + raw['labels'] + end + + def state + raw['state'] == 'closed' ? 'closed' : 'opened' + end + + def has_comments? + raw['comments'] > 0 + end + + def has_labels? + labels.count > 0 + end + + def pull_request? + raw['pull_request'].present? + end + end + end +end diff --git a/lib/github/representation/label.rb b/lib/github/representation/label.rb new file mode 100644 index 00000000000..60aa51f9569 --- /dev/null +++ b/lib/github/representation/label.rb @@ -0,0 +1,13 @@ +module Github + module Representation + class Label < Representation::Base + def color + "##{raw['color']}" + end + + def title + raw['name'] + end + end + end +end diff --git a/lib/github/representation/milestone.rb b/lib/github/representation/milestone.rb new file mode 100644 index 00000000000..917e6394ad4 --- /dev/null +++ b/lib/github/representation/milestone.rb @@ -0,0 +1,25 @@ +module Github + module Representation + class Milestone < Representation::Base + def iid + raw['number'] + end + + def title + raw['title'] + end + + def description + raw['description'] + end + + def due_date + raw['due_on'] + end + + def state + raw['state'] == 'closed' ? 'closed' : 'active' + end + end + end +end diff --git a/lib/github/representation/pull_request.rb b/lib/github/representation/pull_request.rb new file mode 100644 index 00000000000..ac9c8283b4b --- /dev/null +++ b/lib/github/representation/pull_request.rb @@ -0,0 +1,78 @@ +module Github + module Representation + class PullRequest < Representation::Issuable + attr_reader :project + + delegate :user, :repo, :ref, :sha, to: :source_branch, prefix: true + delegate :user, :exists?, :repo, :ref, :sha, :short_sha, to: :target_branch, prefix: true + + def source_project + project + end + + def source_branch_exists? + !cross_project? && source_branch.exists? + end + + def source_branch_name + @source_branch_name ||= + if cross_project? || !source_branch_exists? + source_branch_name_prefixed + else + source_branch_ref + end + end + + def target_project + project + end + + def target_branch_name + @target_branch_name ||= target_branch_exists? ? target_branch_ref : target_branch_name_prefixed + end + + def state + return 'merged' if raw['state'] == 'closed' && raw['merged_at'].present? + return 'closed' if raw['state'] == 'closed' + + 'opened' + end + + def opened? + state == 'opened' + end + + def valid? + source_branch.valid? && target_branch.valid? + end + + private + + def project + @project ||= options.fetch(:project) + end + + def source_branch + @source_branch ||= Representation::Branch.new(raw['head'], repository: project.repository) + end + + def source_branch_name_prefixed + "gh-#{target_branch_short_sha}/#{iid}/#{source_branch_user}/#{source_branch_ref}" + end + + def target_branch + @target_branch ||= Representation::Branch.new(raw['base'], repository: project.repository) + end + + def target_branch_name_prefixed + "gl-#{target_branch_short_sha}/#{iid}/#{target_branch_user}/#{target_branch_ref}" + end + + def cross_project? + return true if source_branch_repo.nil? + + source_branch_repo.id != target_branch_repo.id + end + end + end +end diff --git a/lib/github/representation/release.rb b/lib/github/representation/release.rb new file mode 100644 index 00000000000..e7e4b428c1a --- /dev/null +++ b/lib/github/representation/release.rb @@ -0,0 +1,17 @@ +module Github + module Representation + class Release < Representation::Base + def description + raw['body'] + end + + def tag + raw['tag_name'] + end + + def valid? + !raw['draft'] + end + end + end +end diff --git a/lib/github/representation/repo.rb b/lib/github/representation/repo.rb new file mode 100644 index 00000000000..6938aa7db05 --- /dev/null +++ b/lib/github/representation/repo.rb @@ -0,0 +1,6 @@ +module Github + module Representation + class Repo < Representation::Base + end + end +end diff --git a/lib/github/representation/user.rb b/lib/github/representation/user.rb new file mode 100644 index 00000000000..18591380e25 --- /dev/null +++ b/lib/github/representation/user.rb @@ -0,0 +1,15 @@ +module Github + module Representation + class User < Representation::Base + def email + return @email if defined?(@email) + + @email = Github::User.new(username, options).get.fetch('email', nil) + end + + def username + raw['login'] + end + end + end +end diff --git a/lib/github/response.rb b/lib/github/response.rb new file mode 100644 index 00000000000..761c524b553 --- /dev/null +++ b/lib/github/response.rb @@ -0,0 +1,25 @@ +module Github + class Response + attr_reader :raw, :headers, :status + + def initialize(response) + @raw = response + @headers = response.headers + @status = response.status + end + + def body + Oj.load(raw.body, class_cache: false, mode: :compat) + end + + def rels + links = headers['Link'].to_s.split(', ').map do |link| + href, name = link.match(/<(.*?)>; rel="(\w+)"/).captures + + [name.to_sym, href] + end + + Hash[*links.flatten] + end + end +end diff --git a/lib/github/user.rb b/lib/github/user.rb new file mode 100644 index 00000000000..f88a29e590b --- /dev/null +++ b/lib/github/user.rb @@ -0,0 +1,24 @@ +module Github + class User + attr_reader :username, :options + + def initialize(username, options) + @username = username + @options = options + end + + def get + client.get(user_url).body + end + + private + + def client + @client ||= Github::Client.new(options) + end + + def user_url + "/users/#{username}" + end + end +end diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index d575367d81a..fba80c7132e 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -14,28 +14,16 @@ module Gitlab # Public: Converts the provided Asciidoc markup into HTML. # # input - the source text in Asciidoc format - # context - a Hash with the template context: - # :commit - # :project - # :project_wiki - # :requested_path - # :ref - # asciidoc_opts - a Hash of options to pass to the Asciidoctor converter # - def self.render(input, context, asciidoc_opts = {}) - asciidoc_opts.reverse_merge!( - safe: :secure, - backend: :gitlab_html5, - attributes: [] - ) - asciidoc_opts[:attributes].unshift(*DEFAULT_ADOC_ATTRS) + def self.render(input) + asciidoc_opts = { safe: :secure, + backend: :gitlab_html5, + attributes: DEFAULT_ADOC_ATTRS } plantuml_setup html = ::Asciidoctor.convert(input, asciidoc_opts) - html = Banzai.post_process(html, context) - filter = Banzai::Filter::SanitizationFilter.new(html) html = filter.call.to_s diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index eee5601b0ed..ea918b23a63 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -108,7 +108,7 @@ module Gitlab token = Doorkeeper::AccessToken.by_token(password) if valid_oauth_token?(token) user = User.find_by(id: token.resource_owner_id) - Gitlab::Auth::Result.new(user, nil, :oauth, read_authentication_abilities) + Gitlab::Auth::Result.new(user, nil, :oauth, full_authentication_abilities) end end end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index b358f2efa4f..4fc9a075edc 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -15,18 +15,51 @@ module Gitlab end end + def self.load_in_batch_for_projects(projects) + cached_results_for_projects(projects).zip(projects).each do |result, project| + project.pipeline_status = new(project, result) + project.pipeline_status.load_status + end + end + + def self.cached_results_for_projects(projects) + result = Gitlab::Redis.with do |redis| + redis.multi do + projects.each do |project| + cache_key = cache_key_for_project(project) + redis.exists(cache_key) + redis.hmget(cache_key, :sha, :status, :ref) + end + end + end + + result.each_slice(2).map do |(cache_key_exists, (sha, status, ref))| + pipeline_info = { sha: sha, status: status, ref: ref } + { loaded_from_cache: cache_key_exists, pipeline_info: pipeline_info } + end + end + + def self.cache_key_for_project(project) + "projects/#{project.id}/pipeline_status" + end + def self.update_for_pipeline(pipeline) - new(pipeline.project, - sha: pipeline.sha, - status: pipeline.status, - ref: pipeline.ref).store_in_cache_if_needed + pipeline_info = { + sha: pipeline.sha, + status: pipeline.status, + ref: pipeline.ref + } + + new(pipeline.project, pipeline_info: pipeline_info). + store_in_cache_if_needed end - def initialize(project, sha: nil, status: nil, ref: nil) + def initialize(project, pipeline_info: {}, loaded_from_cache: nil) @project = project - @sha = sha - @ref = ref - @status = status + @sha = pipeline_info[:sha] + @ref = pipeline_info[:ref] + @status = pipeline_info[:status] + @loaded = loaded_from_cache end def has_status? @@ -85,6 +118,8 @@ module Gitlab end def has_cache? + return self.loaded unless self.loaded.nil? + Gitlab::Redis.with do |redis| redis.exists(cache_key) end @@ -95,7 +130,7 @@ module Gitlab end def cache_key - "projects/#{project.id}/build_status" + self.class.cache_key_for_project(project) end end end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index d76aa38f741..1ff34553f0a 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -41,7 +41,7 @@ module Gitlab type = Gitlab::Git.tag_ref?(ref) ? 'tag_push' : 'push' # Hash to be passed as post_receive_data - data = { + { object_kind: type, event_name: type, before: oldrev, @@ -61,16 +61,15 @@ module Gitlab repository: project.hook_attrs.slice(:name, :url, :description, :homepage, :git_http_url, :git_ssh_url, :visibility_level) } - - data end # This method provide a sample data generated with # existing project and commits to test webhooks def build_sample(project, user) - commits = project.repository.commits(project.default_branch, limit: 3) ref = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{project.default_branch}" - build(project, user, commits.last.id, commits.first.id, ref, commits) + commits = project.repository.commits(project.default_branch.to_s, limit: 3) rescue [] + + build(project, user, commits.last&.id, commits.first&.id, ref, commits) end def checkout_sha(repository, newrev, ref) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 6dabbe0264c..298b1a1f4e6 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -498,6 +498,29 @@ module Gitlab columns(table).find { |column| column.name == name } end + + # This will replace the first occurance of a string in a column with + # the replacement + # On postgresql we can use `regexp_replace` for that. + # On mysql we find the location of the pattern, and overwrite it + # with the replacement + def replace_sql(column, pattern, replacement) + quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) + quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) + + if Database.mysql? + locate = Arel::Nodes::NamedFunction. + new('locate', [quoted_pattern, column]) + insert_in_place = Arel::Nodes::NamedFunction. + new('insert', [column, locate, pattern.size, quoted_replacement]) + + Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) + else + replace = Arel::Nodes::NamedFunction. + new("regexp_replace", [column, quoted_pattern, quoted_replacement]) + Arel::Nodes::SqlLiteral.new(replace.to_sql) + end + end end end end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb new file mode 100644 index 00000000000..89530082cd2 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1.rb @@ -0,0 +1,35 @@ +# This module can be included in migrations to make it easier to rename paths +# of `Namespace` & `Project` models certain paths would become `reserved`. +# +# If the way things are stored on the filesystem related to namespaces and +# projects ever changes. Don't update this module, or anything nested in `V1`, +# since it needs to keep functioning for all migrations using it using the state +# that the data is in at the time. Instead, create a `V2` module that implements +# the new way of reserving paths. +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + def self.included(kls) + kls.include(MigrationHelpers) + end + + def rename_wildcard_paths(one_or_more_paths) + rename_child_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + RenameProjects.new(paths, self).rename_projects + end + + def rename_child_paths(one_or_more_paths) + paths = Array(one_or_more_paths) + RenameNamespaces.new(paths, self).rename_namespaces(type: :child) + end + + def rename_root_paths(paths) + paths = Array(paths) + RenameNamespaces.new(paths, self).rename_namespaces(type: :top_level) + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb new file mode 100644 index 00000000000..4fdcb682c2f --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/migration_classes.rb @@ -0,0 +1,76 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + module MigrationClasses + module Routable + def full_path + if route && route.path.present? + @full_path ||= route.path + else + update_route if persisted? + + build_full_path + end + end + + def build_full_path + if parent && path + parent.full_path + '/' + path + else + path + end + end + + def update_route + prepare_route + route.save + end + + def prepare_route + route || build_route(source: self) + route.path = build_full_path + @full_path = nil + end + end + + class Namespace < ActiveRecord::Base + include MigrationClasses::Routable + self.table_name = 'namespaces' + belongs_to :parent, + class_name: "#{MigrationClasses.name}::Namespace" + has_one :route, as: :source + has_many :children, + class_name: "#{MigrationClasses.name}::Namespace", + foreign_key: :parent_id + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Namespace' + end + end + + class Route < ActiveRecord::Base + self.table_name = 'routes' + belongs_to :source, polymorphic: true + end + + class Project < ActiveRecord::Base + include MigrationClasses::Routable + has_one :route, as: :source + self.table_name = 'projects' + + def repository_storage_path + Gitlab.config.repositories.storages[repository_storage]['path'] + end + + # Overridden to have the correct `source_type` for the `route` relation + def self.name + 'Project' + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb new file mode 100644 index 00000000000..de4e6e7c404 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_base.rb @@ -0,0 +1,131 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameBase + attr_reader :paths, :migration + + delegate :update_column_in_batches, + :replace_sql, + to: :migration + + def initialize(paths, migration) + @paths = paths + @migration = migration + end + + def path_patterns + @path_patterns ||= paths.map { |path| "%#{path}" } + end + + def rename_path_for_routable(routable) + old_path = routable.path + old_full_path = routable.full_path + # Only remove the last occurrence of the path name to get the parent namespace path + namespace_path = remove_last_occurrence(old_full_path, old_path) + new_path = rename_path(namespace_path, old_path) + new_full_path = join_routable_path(namespace_path, new_path) + + # skips callbacks & validations + routable.class.where(id: routable). + update_all(path: new_path) + + rename_routes(old_full_path, new_full_path) + + [old_full_path, new_full_path] + end + + def rename_routes(old_full_path, new_full_path) + replace_statement = replace_sql(Route.arel_table[:path], + old_full_path, + new_full_path) + + update_column_in_batches(:routes, :path, replace_statement) do |table, query| + query.where(MigrationClasses::Route.arel_table[:path].matches("#{old_full_path}%")) + end + end + + def rename_path(namespace_path, path_was) + counter = 0 + path = "#{path_was}#{counter}" + + while route_exists?(join_routable_path(namespace_path, path)) + counter += 1 + path = "#{path_was}#{counter}" + end + + path + end + + def remove_last_occurrence(string, pattern) + string.reverse.sub(pattern.reverse, "").reverse + end + + def join_routable_path(namespace_path, top_level) + if namespace_path.present? + File.join(namespace_path, top_level) + else + top_level + end + end + + def route_exists?(full_path) + MigrationClasses::Route.where(Route.arel_table[:path].matches(full_path)).any? + end + + def move_pages(old_path, new_path) + move_folders(pages_dir, old_path, new_path) + end + + def move_uploads(old_path, new_path) + return unless file_storage? + + move_folders(uploads_dir, old_path, new_path) + end + + def move_folders(directory, old_relative_path, new_relative_path) + old_path = File.join(directory, old_relative_path) + return unless File.directory?(old_path) + + new_path = File.join(directory, new_relative_path) + FileUtils.mv(old_path, new_path) + end + + def remove_cached_html_for_projects(project_ids) + update_column_in_batches(:projects, :description_html, nil) do |table, query| + query.where(table[:id].in(project_ids)) + end + + update_column_in_batches(:issues, :description_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + + update_column_in_batches(:merge_requests, :description_html, nil) do |table, query| + query.where(table[:target_project_id].in(project_ids)) + end + + update_column_in_batches(:notes, :note_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + + update_column_in_batches(:milestones, :description_html, nil) do |table, query| + query.where(table[:project_id].in(project_ids)) + end + end + + def file_storage? + CarrierWave::Uploader::Base.storage == CarrierWave::Storage::File + end + + def uploads_dir + File.join(CarrierWave.root, "uploads") + end + + def pages_dir + Settings.pages.path + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb new file mode 100644 index 00000000000..b9f4f3cff3c --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_namespaces.rb @@ -0,0 +1,72 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameNamespaces < RenameBase + include Gitlab::ShellAdapter + + def rename_namespaces(type:) + namespaces_for_paths(type: type).each do |namespace| + rename_namespace(namespace) + end + end + + def namespaces_for_paths(type:) + namespaces = case type + when :child + MigrationClasses::Namespace.where.not(parent_id: nil) + when :top_level + MigrationClasses::Namespace.where(parent_id: nil) + end + with_paths = MigrationClasses::Route.arel_table[:path]. + matches_any(path_patterns) + namespaces.joins(:route).where(with_paths) + end + + def rename_namespace(namespace) + old_full_path, new_full_path = rename_path_for_routable(namespace) + + move_repositories(namespace, old_full_path, new_full_path) + move_uploads(old_full_path, new_full_path) + move_pages(old_full_path, new_full_path) + remove_cached_html_for_projects(projects_for_namespace(namespace).map(&:id)) + end + + def move_repositories(namespace, old_full_path, new_full_path) + repo_paths_for_namespace(namespace).each do |repository_storage_path| + # Ensure old directory exists before moving it + gitlab_shell.add_namespace(repository_storage_path, old_full_path) + + unless gitlab_shell.mv_namespace(repository_storage_path, old_full_path, new_full_path) + message = "Exception moving path #{repository_storage_path} \ + from #{old_full_path} to #{new_full_path}" + Rails.logger.error message + end + end + end + + def repo_paths_for_namespace(namespace) + projects_for_namespace(namespace).distinct.select(:repository_storage). + map(&:repository_storage_path) + end + + def projects_for_namespace(namespace) + namespace_ids = child_ids_for_parent(namespace, ids: [namespace.id]) + namespace_or_children = MigrationClasses::Project. + arel_table[:namespace_id]. + in(namespace_ids) + MigrationClasses::Project.where(namespace_or_children) + end + + def child_ids_for_parent(namespace, ids: []) + namespace.children.each do |child| + ids << child.id + child_ids_for_parent(child, ids: ids) if child.children.any? + end + ids + end + end + end + end + end +end diff --git a/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb new file mode 100644 index 00000000000..448717eb744 --- /dev/null +++ b/lib/gitlab/database/rename_reserved_paths_migration/v1/rename_projects.rb @@ -0,0 +1,45 @@ +module Gitlab + module Database + module RenameReservedPathsMigration + module V1 + class RenameProjects < RenameBase + include Gitlab::ShellAdapter + + def rename_projects + projects_for_paths.each do |project| + rename_project(project) + end + + remove_cached_html_for_projects(projects_for_paths.map(&:id)) + end + + def rename_project(project) + old_full_path, new_full_path = rename_path_for_routable(project) + + move_repository(project, old_full_path, new_full_path) + move_repository(project, "#{old_full_path}.wiki", "#{new_full_path}.wiki") + move_uploads(old_full_path, new_full_path) + move_pages(old_full_path, new_full_path) + end + + def move_repository(project, old_path, new_path) + unless gitlab_shell.mv_repository(project.repository_storage_path, + old_path, + new_path) + Rails.logger.error "Error moving #{old_path} to #{new_path}" + end + end + + def projects_for_paths + return @projects_for_paths if @projects_for_paths + + with_paths = MigrationClasses::Route.arel_table[:path] + .matches_any(path_patterns) + + @projects_for_paths = MigrationClasses::Project.joins(:route).where(with_paths) + end + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 329d12f13d1..0bd226ef050 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -15,6 +15,10 @@ module Gitlab super.tap { |_| store_highlight_cache } end + def real_size + @merge_request_diff.real_size + end + private # Extracted method to highlight in the same iteration to the diff_collection. diff --git a/lib/gitlab/diff/position_tracer.rb b/lib/gitlab/diff/position_tracer.rb index 4d04f867268..c7542a8fabc 100644 --- a/lib/gitlab/diff/position_tracer.rb +++ b/lib/gitlab/diff/position_tracer.rb @@ -82,7 +82,7 @@ module Gitlab file_diff, old_line, new_line = results - Position.new( + new_position = Position.new( old_path: file_diff.old_path, new_path: file_diff.new_path, head_sha: new_diff_refs.head_sha, @@ -91,6 +91,13 @@ module Gitlab old_line: old_line, new_line: new_line ) + + # If a position is found, but is not actually contained in the diff, for example + # because it was an unchanged line in the context of a change that was undone, + # we cannot return this as a successful trace. + return unless new_position.diff_line(repository) + + new_position end private diff --git a/lib/gitlab/email/handler/base_handler.rb b/lib/gitlab/email/handler/base_handler.rb index 3f6ace0311a..0bba433d04b 100644 --- a/lib/gitlab/email/handler/base_handler.rb +++ b/lib/gitlab/email/handler/base_handler.rb @@ -16,6 +16,10 @@ module Gitlab def execute raise NotImplementedError end + + def metrics_params + { handler: self.class.name } + end end end end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index b8ec9138c10..e7f91607e7e 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -1,4 +1,3 @@ - require 'gitlab/email/handler/base_handler' module Gitlab @@ -37,6 +36,10 @@ module Gitlab @project ||= Project.find_by_full_path(project_path) end + def metrics_params + super.merge(project: project) + end + private def create_issue diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index 0e22f2189ee..31bb775c357 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -7,6 +7,8 @@ module Gitlab class CreateNoteHandler < BaseHandler include ReplyProcessing + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+\z/ end @@ -26,16 +28,16 @@ module Gitlab record_name: 'comment') end + def metrics_params + super.merge(project: project) + end + private def author sent_notification.recipient end - def project - sent_notification.project - end - def sent_notification @sent_notification ||= SentNotification.for(mail_key) end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 97d7a8d65ff..df70a063330 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -4,6 +4,8 @@ module Gitlab module Email module Handler class UnsubscribeHandler < BaseHandler + delegate :project, to: :sent_notification, allow_nil: true + def can_handle? mail_key =~ /\A\w+#{Regexp.escape(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX)}\z/ end @@ -17,6 +19,10 @@ module Gitlab noteable.unsubscribe(sent_notification.recipient) end + def metrics_params + super.merge(project: project) + end + private def sent_notification diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index ec0529b5a4b..c270c0ea9ff 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -1,4 +1,3 @@ - require_dependency 'gitlab/email/handler' # Inspired in great part by Discourse's Email::Receiver @@ -32,6 +31,8 @@ module Gitlab raise UnknownIncomingEmail unless handler + Gitlab::Metrics.add_event(:receive_email, handler.metrics_params) + handler.execute end @@ -69,6 +70,8 @@ module Gitlab # Handle emails from clients which append with commas, # example clients are Microsoft exchange and iOS app Gitlab::IncomingEmail.scan_fallback_references(references) + when nil + [] end end diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb index f6e4f279c06..aac210f19e8 100644 --- a/lib/gitlab/etag_caching/router.rb +++ b/lib/gitlab/etag_caching/router.rb @@ -2,31 +2,39 @@ module Gitlab module EtagCaching class Router Route = Struct.new(:regexp, :name) - - RESERVED_WORDS = NamespaceValidator::WILDCARD_ROUTES.map { |word| "/#{word}/" }.join('|') + # We enable an ETag for every request matching the regex. + # To match a regex the path needs to match the following: + # - Don't contain a reserved word (expect for the words used in the + # regex itself) + # - Ending in `noteable/issue/<id>/notes` for the `issue_notes` route + # - Ending in `issues/id`/rendered_title` for the `issue_title` route + USED_IN_ROUTES = %w[noteable issue notes issues rendered_title + commit pipelines merge_requests new].freeze + RESERVED_WORDS = DynamicPathValidator::WILDCARD_ROUTES - USED_IN_ROUTES + RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS) ROUTES = [ Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/noteable/issue/\d+/notes\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z), 'issue_notes' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/issues/\d+/rendered_title\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/issues/\d+/rendered_title\z), 'issue_title' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/commit/\S+/pipelines\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/commit/\S+/pipelines\.json\z), 'commit_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/merge_requests/new\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/new\.json\z), 'new_merge_request_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/merge_requests/\d+/pipelines\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/merge_requests/\d+/pipelines\.json\z), 'merge_request_pipelines' ), Gitlab::EtagCaching::Router::Route.new( - %r(^(?!.*(#{RESERVED_WORDS})).*/pipelines\.json\z), + %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/pipelines\.json\z), 'project_pipelines' ) ].freeze diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index 98fd4e78126..e8bb9e1f805 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -109,10 +109,6 @@ module Gitlab @binary.nil? ? super : @binary == true end - def empty? - !data || data == '' - end - def data encode! @data end diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb index e57d228e688..f918074cb14 100644 --- a/lib/gitlab/git/encoding_helper.rb +++ b/lib/gitlab/git/encoding_helper.rb @@ -40,7 +40,13 @@ module Gitlab def encode_utf8(message) detect = CharlockHolmes::EncodingDetector.detect(message) if detect - CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + begin + CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8') + rescue ArgumentError => e + Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}") + + '' + end else clean(message) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index d7dac9f6149..c3f0de76d01 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -122,13 +122,30 @@ module Gitlab # Returns the number of valid branches def branch_count - rugged.branches.count do |ref| - begin - ref.name && ref.target # ensures the branch is valid + Gitlab::GitalyClient.migrate(:branch_names) do |is_enabled| + if is_enabled + gitaly_ref_client.count_branch_names + else + rugged.branches.count do |ref| + begin + ref.name && ref.target # ensures the branch is valid - true - rescue Rugged::ReferenceError - false + true + rescue Rugged::ReferenceError + false + end + end + end + end + end + + # Returns the number of valid tags + def tag_count + Gitlab::GitalyClient.migrate(:tag_names) do |is_enabled| + if is_enabled + gitaly_ref_client.count_tag_names + else + rugged.tags.count end end end @@ -451,7 +468,7 @@ module Gitlab # Returns true is +from+ is direct ancestor to +to+, otherwise false def is_ancestor?(from, to) - Gitlab::GitalyClient::Commit.is_ancestor(self, from, to) + gitaly_commit_client.is_ancestor(from, to) end # Return an array of Diff objects that represent the diff @@ -494,7 +511,9 @@ module Gitlab # :contains is the commit contained by the refs from which to begin (SHA1 or name) # :max_count is the maximum number of commits to fetch # :skip is the number of commits to skip - # :order is the commits order and allowed value is :date(default) or :topo + # :order is the commits order and allowed value is :none (default), :date, or :topo + # commit ordering types are documented here: + # http://www.rubydoc.info/github/libgit2/rugged/Rugged#SORT_NONE-constant) # def find_commits(options = {}) actual_options = options.dup @@ -522,11 +541,8 @@ module Gitlab end end - if actual_options[:order] == :topo - walker.sorting(Rugged::SORT_TOPO) - else - walker.sorting(Rugged::SORT_NONE) - end + sort_type = rugged_sort_type(actual_options[:order]) + walker.sorting(sort_type) commits = [] offset = actual_options[:skip] @@ -1273,6 +1289,22 @@ module Gitlab def gitaly_ref_client @gitaly_ref_client ||= Gitlab::GitalyClient::Ref.new(self) end + + def gitaly_commit_client + @gitaly_commit_client ||= Gitlab::GitalyClient::Commit.new(self) + end + + # Returns the `Rugged` sorting type constant for a given + # sort type key. Valid keys are `:none`, `:topo`, and `:date` + def rugged_sort_type(key) + @rugged_sort_types ||= { + none: Rugged::SORT_NONE, + topo: Rugged::SORT_TOPO, + date: Rugged::SORT_DATE + } + + @rugged_sort_types.fetch(key, Rugged::SORT_NONE) + end end end end diff --git a/lib/gitlab/gitaly_client/commit.rb b/lib/gitlab/gitaly_client/commit.rb index b7f39f3ef0b..27db1e19bc1 100644 --- a/lib/gitlab/gitaly_client/commit.rb +++ b/lib/gitlab/gitaly_client/commit.rb @@ -5,6 +5,23 @@ module Gitlab # See http://stackoverflow.com/a/40884093/1856239 and https://github.com/git/git/blob/3ad8b5bf26362ac67c9020bf8c30eee54a84f56d/cache.h#L1011-L1012 EMPTY_TREE_ID = '4b825dc642cb6eb9a060e54bf8d69288fbee4904'.freeze + attr_accessor :stub + + def initialize(repository) + @gitaly_repo = repository.gitaly_repository + @stub = Gitaly::Commit::Stub.new(nil, nil, channel_override: repository.gitaly_channel) + end + + def is_ancestor(ancestor_id, child_id) + request = Gitaly::CommitIsAncestorRequest.new( + repository: @gitaly_repo, + ancestor_id: ancestor_id, + child_id: child_id + ) + + @stub.commit_is_ancestor(request).value + end + class << self def diff_from_parent(commit, options = {}) repository = commit.project.repository @@ -20,18 +37,6 @@ module Gitlab Gitlab::Git::DiffCollection.new(stub.commit_diff(request), options) end - - def is_ancestor(repository, ancestor_id, child_id) - gitaly_repo = repository.gitaly_repository - stub = Gitaly::Commit::Stub.new(nil, nil, channel_override: repository.gitaly_channel) - request = Gitaly::CommitIsAncestorRequest.new( - repository: gitaly_repo, - ancestor_id: ancestor_id, - child_id: child_id - ) - - stub.commit_is_ancestor(request).value - end end end end diff --git a/lib/gitlab/gitaly_client/ref.rb b/lib/gitlab/gitaly_client/ref.rb index d3c0743db4e..2a5e8f73e55 100644 --- a/lib/gitlab/gitaly_client/ref.rb +++ b/lib/gitlab/gitaly_client/ref.rb @@ -34,6 +34,14 @@ module Gitlab stub.find_ref_name(request).name end + def count_tag_names + tag_names.count + end + + def count_branch_names + branch_names.count + end + private def consume_refs_response(response, prefix:) diff --git a/lib/gitlab/google_code_import/importer.rb b/lib/gitlab/google_code_import/importer.rb index b02b9737493..5ca3e6a95ca 100644 --- a/lib/gitlab/google_code_import/importer.rb +++ b/lib/gitlab/google_code_import/importer.rb @@ -1,7 +1,23 @@ module Gitlab module GoogleCodeImport class Importer - attr_reader :project, :repo + attr_reader :project, :repo, :closed_statuses + + NICE_LABEL_COLOR_HASH = + { + 'Status: New' => '#428bca', + 'Status: Accepted' => '#5cb85c', + 'Status: Started' => '#8e44ad', + 'Priority: Critical' => '#ffcfcf', + 'Priority: High' => '#deffcf', + 'Priority: Medium' => '#fff5cc', + 'Priority: Low' => '#cfe9ff', + 'Type: Defect' => '#d9534f', + 'Type: Enhancement' => '#44ad8e', + 'Type: Task' => '#4b6dd0', + 'Type: Review' => '#8e44ad', + 'Type: Other' => '#7f8c8d' + }.freeze def initialize(project) @project = project @@ -161,45 +177,19 @@ module Gitlab end def nice_label_color(name) - case name - when /\AComponent:/ - "#fff39e" - when /\AOpSys:/ - "#e2e2e2" - when /\AMilestone:/ - "#fee3ff" - - when "Status: New" - "#428bca" - when "Status: Accepted" - "#5cb85c" - when "Status: Started" - "#8e44ad" - - when "Priority: Critical" - "#ffcfcf" - when "Priority: High" - "#deffcf" - when "Priority: Medium" - "#fff5cc" - when "Priority: Low" - "#cfe9ff" - - when "Type: Defect" - "#d9534f" - when "Type: Enhancement" - "#44ad8e" - when "Type: Task" - "#4b6dd0" - when "Type: Review" - "#8e44ad" - when "Type: Other" - "#7f8c8d" - when *@closed_statuses.map { |s| nice_status_name(s) } - "#cfcfcf" - else - "#e2e2e2" - end + NICE_LABEL_COLOR_HASH[name] || + case name + when /\AComponent:/ + '#fff39e' + when /\AOpSys:/ + '#e2e2e2' + when /\AMilestone:/ + '#fee3ff' + when *closed_statuses.map { |s| nice_status_name(s) } + '#cfcfcf' + else + '#e2e2e2' + end end def nice_label_name(name) diff --git a/lib/gitlab/issuable_sorter.rb b/lib/gitlab/issuable_sorter.rb new file mode 100644 index 00000000000..d392214867a --- /dev/null +++ b/lib/gitlab/issuable_sorter.rb @@ -0,0 +1,29 @@ +module Gitlab + module IssuableSorter + class << self + def sort(project, issuables, &sort_key) + grouped_items = issuables.group_by do |issuable| + if issuable.project.id == project.id + :project_ref + elsif issuable.project.namespace.id == project.namespace.id + :namespace_ref + else + :full_ref + end + end + + natural_sort_issuables(grouped_items[:project_ref], project) + + natural_sort_issuables(grouped_items[:namespace_ref], project) + + natural_sort_issuables(grouped_items[:full_ref], project) + end + + private + + def natural_sort_issuables(issuables, project) + VersionSorter.sort(issuables || []) do |issuable| + issuable.to_reference(project) + end + end + end + end +end diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb index 857e0abf710..c6dfa4ad9bd 100644 --- a/lib/gitlab/metrics.rb +++ b/lib/gitlab/metrics.rb @@ -138,6 +138,11 @@ module Gitlab @series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_' end + # Allow access from other metrics related middlewares + def self.current_transaction + Transaction.current + end + # When enabled this should be set before being used as the usual pattern # "@foo ||= bar" is _not_ thread-safe. if enabled? @@ -149,10 +154,5 @@ module Gitlab new(udp: { host: host, port: port }) end end - - # Allow access from other metrics related middlewares - def self.current_transaction - Transaction.current - end end end diff --git a/lib/gitlab/o_auth/user.rb b/lib/gitlab/o_auth/user.rb index 6e42d8941fb..afd24b4dcc5 100644 --- a/lib/gitlab/o_auth/user.rb +++ b/lib/gitlab/o_auth/user.rb @@ -148,7 +148,7 @@ module Gitlab def build_new_user user_params = user_attributes.merge(extern_uid: auth_hash.uid, provider: auth_hash.provider, skip_confirmation: true) - Users::BuildService.new(nil, user_params).execute + Users::BuildService.new(nil, user_params).execute(skip_authorization: true) end def user_attributes diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index e67acf28c94..c2adc9aa10b 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -4,19 +4,11 @@ module Gitlab # Public: Converts the provided markup into HTML. # # input - the source text in a markup format - # context - a Hash with the template context: - # :commit - # :project - # :project_wiki - # :requested_path - # :ref # - def self.render(file_name, input, context) + def self.render(file_name, input) html = GitHub::Markup.render(file_name, input). force_encoding(input.encoding) - html = Banzai.post_process(html, context) - filter = Banzai::Filter::SanitizationFilter.new(html) html = filter.call.to_s diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 08b061d5e31..b7fef5dd068 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -22,6 +22,10 @@ module Gitlab @namespace_regex ||= /\A#{NAMESPACE_REGEX_STR}\z/.freeze end + def full_namespace_regex + @full_namespace_regex ||= %r{\A#{FULL_NAMESPACE_REGEX_STR}\z} + end + def namespace_route_regex @namespace_route_regex ||= /#{NAMESPACE_REGEX_STR}/.freeze end diff --git a/lib/gitlab/template/dockerfile_template.rb b/lib/gitlab/template/dockerfile_template.rb index d5d3e045a42..20b054b0bd8 100644 --- a/lib/gitlab/template/dockerfile_template.rb +++ b/lib/gitlab/template/dockerfile_template.rb @@ -8,7 +8,7 @@ module Gitlab class << self def extension - 'Dockerfile' + '.Dockerfile' end def categories @@ -18,7 +18,7 @@ module Gitlab end def base_dir - Rails.root.join('vendor/dockerfile') + Rails.root.join('vendor/Dockerfile') end def finder(project = nil) diff --git a/lib/gitlab/user_access.rb b/lib/gitlab/user_access.rb index 54728e5ff0e..e46ff313654 100644 --- a/lib/gitlab/user_access.rb +++ b/lib/gitlab/user_access.rb @@ -44,9 +44,7 @@ module Gitlab if ProtectedBranch.protected?(project, ref) return true if project.empty_repo? && project.user_can_push_to_empty_repo?(user) - has_access = project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push) - - has_access || !project.repository.branch_exists?(ref) && can_merge_to_branch?(ref) + project.protected_branches.protected_ref_accessible_to?(ref, user, action: :push) else user.can?(:push_code, project) end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e6e40f6945d..c551f939df1 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -168,7 +168,7 @@ module Gitlab end def secret_path - Rails.root.join('.gitlab_workhorse_secret') + Gitlab.config.workhorse.secret_file end def set_key_and_notify(key, value, expire: nil, overwrite: true) diff --git a/lib/tasks/brakeman.rake b/lib/tasks/brakeman.rake index 2301ec9b228..99b3168d9eb 100644 --- a/lib/tasks/brakeman.rake +++ b/lib/tasks/brakeman.rake @@ -2,7 +2,7 @@ desc 'Security check via brakeman' task :brakeman do # We get 0 warnings at level 'w3' but we would like to reach 'w2'. Merge # requests are welcome! - if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb -w3 -z)) + if system(*%w(brakeman --no-progress --skip-files lib/backup/repository.rb,app/controllers/unicorn_test_controller.rb -w3 -z)) puts 'Security check succeed' else puts 'Security check failed' diff --git a/lib/tasks/cache.rake b/lib/tasks/cache.rake index d55923673b1..125a3d560d6 100644 --- a/lib/tasks/cache.rake +++ b/lib/tasks/cache.rake @@ -21,12 +21,7 @@ namespace :cache do end end - desc "GitLab | Clear database cache (in the background)" - task db: :environment do - ClearDatabaseCacheWorker.perform_async - end - - task all: [:db, :redis] + task all: [:redis] end task clear: 'cache:clear:redis' diff --git a/lib/tasks/gitlab/gitaly.rake b/lib/tasks/gitlab/gitaly.rake index 8079c6e416c..3c5bc0146a1 100644 --- a/lib/tasks/gitlab/gitaly.rake +++ b/lib/tasks/gitlab/gitaly.rake @@ -1,21 +1,24 @@ namespace :gitlab do namespace :gitaly do desc "GitLab | Install or upgrade gitaly" - task :install, [:dir] => :environment do |t, args| + task :install, [:dir, :repo] => :environment do |t, args| + require 'toml' + warn_user_is_not_gitlab unless args.dir.present? abort %(Please specify the directory where you want to install gitaly:\n rake "gitlab:gitaly:install[/home/git/gitaly]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitaly.git') version = Gitlab::GitalyClient.expected_server_version - repo = 'https://gitlab.com/gitlab-org/gitaly.git' - checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' Dir.chdir(args.dir) do + create_gitaly_configuration run_command!([command]) end end @@ -33,5 +36,39 @@ namespace :gitlab do puts TOML.dump(storage: config) end + + private + + # We cannot create config.toml files for all possible Gitaly configuations. + # For instance, if Gitaly is running on another machine then it makes no + # sense to write a config.toml file on the current machine. This method will + # only write a config.toml file in the most common and simplest case: the + # case where we have exactly one Gitaly process and we are sure it is + # running locally because it uses a Unix socket. + def create_gitaly_configuration + storages = [] + address = nil + + Gitlab.config.repositories.storages.each do |key, val| + if address + if address != val['gitaly_address'] + raise ArgumentError, "Your gitlab.yml contains more than one gitaly_address." + end + elsif URI(val['gitaly_address']).scheme != 'unix' + raise ArgumentError, "Automatic config.toml generation only supports 'unix:' addresses." + else + address = val['gitaly_address'] + end + + storages << { name: key, path: val['path'] } + end + + File.open("config.toml", "w") do |f| + f.puts TOML.dump(socket_path: address.sub(%r{\Aunix:}, ''), storages: storages) + end + rescue ArgumentError => e + puts "Skipping config.toml generation:" + puts e.message + end end end diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake index cb2adc81c9d..1b04e1350ed 100644 --- a/lib/tasks/gitlab/update_templates.rake +++ b/lib/tasks/gitlab/update_templates.rake @@ -5,7 +5,7 @@ namespace :gitlab do end def update(template) - sub_dir = template.repo_url.match(/([a-z-]+)\.git\z/)[1] + sub_dir = template.repo_url.match(/([A-Za-z-]+)\.git\z/)[1] dir = File.join(vendor_directory, sub_dir) unless clone_repository(template.repo_url, dir) @@ -45,7 +45,11 @@ namespace :gitlab do Template.new( "https://gitlab.com/gitlab-org/gitlab-ci-yml.git", /(\.{1,2}|LICENSE|CONTRIBUTING.md|Pages|autodeploy|\.gitlab-ci.yml)\z/ - ) + ), + Template.new( + "https://gitlab.com/gitlab-org/Dockerfile.git", + /(\.{1,2}|LICENSE|CONTRIBUTING.md|\.Dockerfile)\z/ + ), ].freeze def vendor_directory diff --git a/lib/tasks/gitlab/workhorse.rake b/lib/tasks/gitlab/workhorse.rake index a00b02188cf..e7ac0b5859f 100644 --- a/lib/tasks/gitlab/workhorse.rake +++ b/lib/tasks/gitlab/workhorse.rake @@ -1,16 +1,16 @@ namespace :gitlab do namespace :workhorse do desc "GitLab | Install or upgrade gitlab-workhorse" - task :install, [:dir] => :environment do |t, args| + task :install, [:dir, :repo] => :environment do |t, args| warn_user_is_not_gitlab unless args.dir.present? abort %(Please specify the directory where you want to install gitlab-workhorse:\n rake "gitlab:workhorse:install[/home/git/gitlab-workhorse]") end + args.with_defaults(repo: 'https://gitlab.com/gitlab-org/gitlab-workhorse.git') version = Gitlab::Workhorse.version - repo = 'https://gitlab.com/gitlab-org/gitlab-workhorse.git' - checkout_or_clone_version(version: version, repo: repo, target_dir: args.dir) + checkout_or_clone_version(version: version, repo: args.repo, target_dir: args.dir) _, status = Gitlab::Popen.popen(%w[which gmake]) command = status.zero? ? 'gmake' : 'make' diff --git a/lib/tasks/import.rake b/lib/tasks/import.rake index a9dad6a1bf0..bc76d7edc55 100644 --- a/lib/tasks/import.rake +++ b/lib/tasks/import.rake @@ -1,67 +1,5 @@ require 'benchmark' require 'rainbow/ext/string' -require_relative '../gitlab/shell_adapter' -require_relative '../gitlab/github_import/importer' - -class NewImporter < ::Gitlab::GithubImport::Importer - def execute - # Same as ::Gitlab::GithubImport::Importer#execute, but showing some progress. - puts 'Importing repository...'.color(:aqua) - import_repository unless project.repository_exists? - - puts 'Importing labels...'.color(:aqua) - import_labels - - puts 'Importing milestones...'.color(:aqua) - import_milestones - - puts 'Importing pull requests...'.color(:aqua) - import_pull_requests - - puts 'Importing issues...'.color(:aqua) - import_issues - - puts 'Importing issue comments...'.color(:aqua) - import_comments(:issues) - - puts 'Importing pull request comments...'.color(:aqua) - import_comments(:pull_requests) - - puts 'Importing wiki...'.color(:aqua) - import_wiki - - # Gitea doesn't have a Release API yet - # See https://github.com/go-gitea/gitea/issues/330 - unless project.gitea_import? - import_releases - end - - handle_errors - - project.repository.after_import - project.import_finish - - true - end - - def import_repository - begin - raise 'Blocked import URL.' if Gitlab::UrlBlocker.blocked_url?(project.import_url) - - project.create_repository - project.repository.add_remote(project.import_type, project.import_url) - project.repository.set_remote_as_mirror(project.import_type) - project.repository.fetch_remote(project.import_type, forced: true) - rescue => e - # Expire cache to prevent scenarios such as: - # 1. First import failed, but the repo was imported successfully, so +exists?+ returns true - # 2. Retried import, repo is broken or not imported but +exists?+ still returns true - project.repository.expire_content_cache if project.repository_exists? - - raise "Error importing repository #{project.import_url} into #{project.path_with_namespace} - #{e.message}" - end - end -end class GithubImport def self.run!(*args) @@ -69,14 +7,14 @@ class GithubImport end def initialize(token, gitlab_username, project_path, extras) - @token = token + @options = { url: 'https://api.github.com', token: token, verbose: true } @project_path = project_path @current_user = User.find_by_username(gitlab_username) @github_repo = extras.empty? ? nil : extras.first end def run! - @repo = GithubRepos.new(@token, @current_user, @github_repo).choose_one! + @repo = GithubRepos.new(@options, @current_user, @github_repo).choose_one! raise 'No repo found!' unless @repo @@ -90,25 +28,24 @@ class GithubImport private def show_warning! - puts "This will import GH #{@repo.full_name.bright} into GL #{@project_path.bright} as #{@current_user.name}" + puts "This will import GitHub #{@repo['full_name'].bright} into GitLab #{@project_path.bright} as #{@current_user.name}" puts "Permission checks are ignored. Press any key to continue.".color(:red) STDIN.getch - puts 'Starting the import...'.color(:green) + puts 'Starting the import (this could take a while)'.color(:green) end def import! - import_url = @project.import_url.gsub(/\:\/\/(.*@)?/, "://#{@token}@") - @project.update(import_url: import_url) - @project.import_start timings = Benchmark.measure do - NewImporter.new(@project).execute + Github::Import.new(@project, @options).execute end puts "Import finished. Timings: #{timings}".color(:green) + + @project.import_finish end def new_project @@ -116,17 +53,17 @@ class GithubImport namespace_path, _sep, name = @project_path.rpartition('/') namespace = find_or_create_namespace(namespace_path) - Project.create!( - import_url: "https://#{@token}@github.com/#{@repo.full_name}.git", + Projects::CreateService.new( + @current_user, name: name, path: name, - description: @repo.description, - namespace: namespace, + description: @repo['description'], + namespace_id: namespace.id, visibility_level: visibility_level, import_type: 'github', - import_source: @repo.full_name, - creator: @current_user - ) + import_source: @repo['full_name'], + skip_wiki: @repo['has_wiki'] + ).execute end end @@ -134,7 +71,6 @@ class GithubImport return @current_user.namespace if names == @current_user.namespace_path return @current_user.namespace unless @current_user.can_create_group? - names = params[:target_namespace].presence || names full_path_namespace = Namespace.find_by_full_path(names) return full_path_namespace if full_path_namespace @@ -159,13 +95,13 @@ class GithubImport end def visibility_level - @repo.private ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility + @repo['private'] ? Gitlab::VisibilityLevel::PRIVATE : current_application_settings.default_project_visibility end end class GithubRepos - def initialize(token, current_user, github_repo) - @token = token + def initialize(options, current_user, github_repo) + @options = options @current_user = current_user @github_repo = github_repo end @@ -174,17 +110,17 @@ class GithubRepos return found_github_repo if @github_repo repos.each do |repo| - print "ID: #{repo[:id].to_s.bright} ".color(:green) - puts "- Name: #{repo[:full_name]}".color(:green) + print "ID: #{repo['id'].to_s.bright}".color(:green) + print "\tName: #{repo['full_name']}\n".color(:green) end print 'ID? '.bright - repos.find { |repo| repo[:id] == repo_id } + repos.find { |repo| repo['id'] == repo_id } end def found_github_repo - repos.find { |repo| repo[:full_name] == @github_repo } + repos.find { |repo| repo['full_name'] == @github_repo } end def repo_id @@ -192,11 +128,7 @@ class GithubRepos end def repos - @repos ||= client.repos - end - - def client - @client ||= Gitlab::GithubImport::Client.new(@token, {}) + Github::Repositories.new(@options).fetch end end |