From 2c5bcf2e1b5b5574238555657296a8831b989d1e Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Wed, 23 Mar 2016 22:36:35 +0100 Subject: Add endpoints for archiving and unarchiving --- lib/api/projects.rb | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index 6fcb5261e40..aa60a39f341 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -244,6 +244,34 @@ module API end end + # Archive project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # PUT /projects/:id/archive + put ':id/archive' do + authorize!(:archive_project, user_project) + + user_project.archive! + + present @project, with: Entities::Project + end + + # Unarchive project + # + # Parameters: + # id (required) - The ID of a project + # Example Request: + # PUT /projects/:id/unarchive + put ':id/unarchive' do + authorize!(:archive_project, user_project) + + user_project.unarchive! + + present @project, with: Entities::Project + end + # Remove project # # Parameters: -- cgit v1.2.3 From 8558483417c75da5d636f65935668c230678e032 Mon Sep 17 00:00:00 2001 From: Robert Speicher Date: Thu, 24 Mar 2016 13:23:47 -0400 Subject: Remove redundant `require`s from Banzai files We're trying to avoid circular dependency errors. --- lib/banzai/filter.rb | 2 -- lib/banzai/filter/autolink_filter.rb | 1 - lib/banzai/filter/emoji_filter.rb | 4 ---- lib/banzai/filter/external_link_filter.rb | 2 -- lib/banzai/filter/gollum_tags_filter.rb | 3 --- lib/banzai/filter/markdown_filter.rb | 2 -- lib/banzai/filter/milestone_reference_filter.rb | 2 -- lib/banzai/filter/redactor_filter.rb | 2 -- lib/banzai/filter/reference_filter.rb | 3 --- lib/banzai/filter/reference_gatherer_filter.rb | 2 -- lib/banzai/filter/relative_link_filter.rb | 1 - lib/banzai/filter/sanitization_filter.rb | 3 --- lib/banzai/filter/syntax_highlight_filter.rb | 1 - lib/banzai/filter/table_of_contents_filter.rb | 2 -- lib/banzai/filter/upload_link_filter.rb | 1 - lib/banzai/filter/yaml_front_matter_filter.rb | 3 --- lib/banzai/pipeline/base_pipeline.rb | 2 -- lib/banzai/pipeline/wiki_pipeline.rb | 2 -- 18 files changed, 38 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter.rb b/lib/banzai/filter.rb index 905c4c0144e..3eb544dfef9 100644 --- a/lib/banzai/filter.rb +++ b/lib/banzai/filter.rb @@ -1,5 +1,3 @@ -require 'active_support/core_ext/string/output_safety' - module Banzai module Filter def self.[](name) diff --git a/lib/banzai/filter/autolink_filter.rb b/lib/banzai/filter/autolink_filter.rb index 856f56fb175..fac7dad3243 100644 --- a/lib/banzai/filter/autolink_filter.rb +++ b/lib/banzai/filter/autolink_filter.rb @@ -1,4 +1,3 @@ -require 'html/pipeline/filter' require 'uri' module Banzai diff --git a/lib/banzai/filter/emoji_filter.rb b/lib/banzai/filter/emoji_filter.rb index 207437ba7cf..d25de900674 100644 --- a/lib/banzai/filter/emoji_filter.rb +++ b/lib/banzai/filter/emoji_filter.rb @@ -1,7 +1,3 @@ -require 'action_controller' -require 'gitlab_emoji' -require 'html/pipeline/filter' - module Banzai module Filter # HTML filter that replaces :emoji: with images. diff --git a/lib/banzai/filter/external_link_filter.rb b/lib/banzai/filter/external_link_filter.rb index 8d368f3b9e7..d179bea181e 100644 --- a/lib/banzai/filter/external_link_filter.rb +++ b/lib/banzai/filter/external_link_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter # HTML Filter to add a `rel="nofollow"` attribute to external links diff --git a/lib/banzai/filter/gollum_tags_filter.rb b/lib/banzai/filter/gollum_tags_filter.rb index f31f921903b..7ce26db1b90 100644 --- a/lib/banzai/filter/gollum_tags_filter.rb +++ b/lib/banzai/filter/gollum_tags_filter.rb @@ -1,6 +1,3 @@ -require 'banzai' -require 'html/pipeline/filter' - module Banzai module Filter # HTML Filter for parsing Gollum's tags in HTML. It's only parses the diff --git a/lib/banzai/filter/markdown_filter.rb b/lib/banzai/filter/markdown_filter.rb index 0659fed1419..9b209533a89 100644 --- a/lib/banzai/filter/markdown_filter.rb +++ b/lib/banzai/filter/markdown_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter class MarkdownFilter < HTML::Pipeline::TextFilter diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index e88b27c1fae..8f710a92bdc 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Filter # HTML filter that replaces milestone references with links. diff --git a/lib/banzai/filter/redactor_filter.rb b/lib/banzai/filter/redactor_filter.rb index 7141ed7c9bd..e589b5df6ec 100644 --- a/lib/banzai/filter/redactor_filter.rb +++ b/lib/banzai/filter/redactor_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter # HTML filter that removes references to records that the current user does diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index 132f0a4bd93..a3326ae042c 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -1,6 +1,3 @@ -require 'active_support/core_ext/string/output_safety' -require 'html/pipeline/filter' - module Banzai module Filter # Base class for GitLab Flavored Markdown reference filters. diff --git a/lib/banzai/filter/reference_gatherer_filter.rb b/lib/banzai/filter/reference_gatherer_filter.rb index 86d484feb90..96fdb06304e 100644 --- a/lib/banzai/filter/reference_gatherer_filter.rb +++ b/lib/banzai/filter/reference_gatherer_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter # HTML filter that gathers all referenced records that the current user has diff --git a/lib/banzai/filter/relative_link_filter.rb b/lib/banzai/filter/relative_link_filter.rb index 41380627d39..ea21c7b041c 100644 --- a/lib/banzai/filter/relative_link_filter.rb +++ b/lib/banzai/filter/relative_link_filter.rb @@ -1,4 +1,3 @@ -require 'html/pipeline/filter' require 'uri' module Banzai diff --git a/lib/banzai/filter/sanitization_filter.rb b/lib/banzai/filter/sanitization_filter.rb index e8011519608..42dbab9d27e 100644 --- a/lib/banzai/filter/sanitization_filter.rb +++ b/lib/banzai/filter/sanitization_filter.rb @@ -1,6 +1,3 @@ -require 'html/pipeline/filter' -require 'html/pipeline/sanitization_filter' - module Banzai module Filter # Sanitize HTML diff --git a/lib/banzai/filter/syntax_highlight_filter.rb b/lib/banzai/filter/syntax_highlight_filter.rb index 8c5855e5ffc..62a79c62e20 100644 --- a/lib/banzai/filter/syntax_highlight_filter.rb +++ b/lib/banzai/filter/syntax_highlight_filter.rb @@ -1,4 +1,3 @@ -require 'html/pipeline/filter' require 'rouge/plugins/redcarpet' module Banzai diff --git a/lib/banzai/filter/table_of_contents_filter.rb b/lib/banzai/filter/table_of_contents_filter.rb index 4056dcd6d64..a4eda6fdf76 100644 --- a/lib/banzai/filter/table_of_contents_filter.rb +++ b/lib/banzai/filter/table_of_contents_filter.rb @@ -1,5 +1,3 @@ -require 'html/pipeline/filter' - module Banzai module Filter # HTML filter that adds an anchor child element to all Headers in a diff --git a/lib/banzai/filter/upload_link_filter.rb b/lib/banzai/filter/upload_link_filter.rb index f642aee0967..7edfe5ade2d 100644 --- a/lib/banzai/filter/upload_link_filter.rb +++ b/lib/banzai/filter/upload_link_filter.rb @@ -1,4 +1,3 @@ -require 'html/pipeline/filter' require 'uri' module Banzai diff --git a/lib/banzai/filter/yaml_front_matter_filter.rb b/lib/banzai/filter/yaml_front_matter_filter.rb index e4e2f3f228d..58e3e81209e 100644 --- a/lib/banzai/filter/yaml_front_matter_filter.rb +++ b/lib/banzai/filter/yaml_front_matter_filter.rb @@ -1,6 +1,3 @@ -require 'html/pipeline/filter' -require 'yaml' - module Banzai module Filter class YamlFrontMatterFilter < HTML::Pipeline::Filter diff --git a/lib/banzai/pipeline/base_pipeline.rb b/lib/banzai/pipeline/base_pipeline.rb index f60966c3c0f..321fd5bbe14 100644 --- a/lib/banzai/pipeline/base_pipeline.rb +++ b/lib/banzai/pipeline/base_pipeline.rb @@ -1,5 +1,3 @@ -require 'html/pipeline' - module Banzai module Pipeline class BasePipeline diff --git a/lib/banzai/pipeline/wiki_pipeline.rb b/lib/banzai/pipeline/wiki_pipeline.rb index 9b4ff0f0f80..0b5a9e0b2b8 100644 --- a/lib/banzai/pipeline/wiki_pipeline.rb +++ b/lib/banzai/pipeline/wiki_pipeline.rb @@ -1,5 +1,3 @@ -require 'banzai' - module Banzai module Pipeline class WikiPipeline < FullPipeline -- cgit v1.2.3 From 3549d7c1d402c10c567c239b006132c45b0c0d1e Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Thu, 24 Mar 2016 13:36:45 +0100 Subject: PUT becomes POST on archiving endpoints Also the specs have a minor improvement. Mainly the access right spec. Changes are reflected in the docs --- lib/api/projects.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/api/projects.rb b/lib/api/projects.rb index aa60a39f341..24b31005475 100644 --- a/lib/api/projects.rb +++ b/lib/api/projects.rb @@ -250,12 +250,12 @@ module API # id (required) - The ID of a project # Example Request: # PUT /projects/:id/archive - put ':id/archive' do + post ':id/archive' do authorize!(:archive_project, user_project) user_project.archive! - present @project, with: Entities::Project + present user_project, with: Entities::Project end # Unarchive project @@ -264,12 +264,12 @@ module API # id (required) - The ID of a project # Example Request: # PUT /projects/:id/unarchive - put ':id/unarchive' do + post ':id/unarchive' do authorize!(:archive_project, user_project) user_project.unarchive! - present @project, with: Entities::Project + present user_project, with: Entities::Project end # Remove project -- cgit v1.2.3 From c8be7f1cf027d27bba50f6fa4fdeaee33e3f531f Mon Sep 17 00:00:00 2001 From: Mariusz Jachimowicz Date: Thu, 24 Mar 2016 18:38:37 +0000 Subject: api - expose label description --- lib/api/entities.rb | 2 +- lib/api/labels.rb | 24 +++++++++++++----------- 2 files changed, 14 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 197e826e5bc..f686c568bee 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -292,7 +292,7 @@ module API end class Label < Grape::Entity - expose :name, :color + expose :name, :color, :description end class Compare < Grape::Entity diff --git a/lib/api/labels.rb b/lib/api/labels.rb index 78ca58ad0d1..4af6bef0fa7 100644 --- a/lib/api/labels.rb +++ b/lib/api/labels.rb @@ -17,17 +17,18 @@ module API # Creates a new label # # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # color (required) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) + # id (required) - The ID of a project + # name (required) - The name of the label to be created + # color (required) - Color of the label given in 6-digit hex + # notation with leading '#' sign (e.g. #FFAABB) + # description (optional) - The description of label to be created # Example Request: # POST /projects/:id/labels post ':id/labels' do authorize! :admin_label, user_project required_attributes! [:name, :color] - attrs = attributes_for_keys [:name, :color] + attrs = attributes_for_keys [:name, :color, :description] label = user_project.find_label(attrs[:name]) conflict!('Label already exists') if label @@ -62,11 +63,12 @@ module API # Updates an existing label. At least one optional parameter is required. # # Parameters: - # id (required) - The ID of a project - # name (required) - The name of the label to be deleted - # new_name (optional) - The new name of the label - # color (optional) - Color of the label given in 6-digit hex - # notation with leading '#' sign (e.g. #FFAABB) + # id (required) - The ID of a project + # name (required) - The name of the label to be deleted + # new_name (optional) - The new name of the label + # color (optional) - Color of the label given in 6-digit hex + # notation with leading '#' sign (e.g. #FFAABB) + # description (optional) - The description of label to be created # Example Request: # PUT /projects/:id/labels put ':id/labels' do @@ -76,7 +78,7 @@ module API label = user_project.find_label(params[:name]) not_found!('Label not found') unless label - attrs = attributes_for_keys [:new_name, :color] + attrs = attributes_for_keys [:new_name, :color, :description] if attrs.empty? render_api_error!('Required parameters "new_name" or "color" ' \ -- cgit v1.2.3 From 31e76baf610e1307090a6bac3a7b3d525bce057a Mon Sep 17 00:00:00 2001 From: David Padilla Date: Mon, 29 Feb 2016 23:29:20 -0600 Subject: Fix #2364. Fall back to In-Reply-To header when reply key not available --- lib/gitlab/email/receiver.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'lib') diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index d4b6f6d120d..d55bacde5b0 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -63,6 +63,10 @@ module Gitlab end def reply_key + key_from_to_address || key_from_in_reply_to_header + end + + def key_from_to_address key = nil message.to.each do |address| key = Gitlab::IncomingEmail.key_from_address(address) @@ -72,6 +76,17 @@ module Gitlab key end + def key_from_in_reply_to_header + reply_key = nil + + message[:in_reply_to].message_ids.each do |message_id| + reply_key = Gitlab::IncomingEmail.key_from_address(message_id) + break if reply_key + end + + reply_key + end + def sent_notification return nil unless reply_key -- cgit v1.2.3 From 9f218fc184894d61c10f738c59bce97780f06e25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 17 Mar 2016 20:03:51 +0100 Subject: Improve and finish the fallback to the In-Reply-To and References header for the reply-by-email feature A few things to note: - The IncomingEmail feature is now enabled even without a correctly-formatted sub-address - Message-ID for new thread mail are kept the same so that subsequent notifications to this thread are grouped in the thread by the email service that receives the notification (i.e. In-Reply-To of the answer == Message-ID of the first thread message) - To maximize our chance to be able to retrieve the reply key, we look for it in the In-Reply-To header and the References header - The pattern for the fallback reply message id is "reply-[key]@[gitlab_host]" - Improve docs thanks to Axil --- lib/gitlab/email/receiver.rb | 10 +++++----- lib/gitlab/incoming_email.rb | 16 ++++++++++------ lib/tasks/gitlab/check.rake | 15 --------------- 3 files changed, 15 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index d55bacde5b0..97ef9851d71 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -63,10 +63,10 @@ module Gitlab end def reply_key - key_from_to_address || key_from_in_reply_to_header + key_from_to_header || key_from_additional_headers end - def key_from_to_address + def key_from_to_header key = nil message.to.each do |address| key = Gitlab::IncomingEmail.key_from_address(address) @@ -76,11 +76,11 @@ module Gitlab key end - def key_from_in_reply_to_header + def key_from_additional_headers reply_key = nil - message[:in_reply_to].message_ids.each do |message_id| - reply_key = Gitlab::IncomingEmail.key_from_address(message_id) + Array(message.references).each do |message_id| + reply_key = Gitlab::IncomingEmail.key_from_fallback_reply_message_id(message_id) break if reply_key end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index 9068d79c95e..8ce9d32abe0 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -1,13 +1,10 @@ module Gitlab module IncomingEmail class << self - def enabled? - config.enabled && address_formatted_correctly? - end + FALLBACK_REPLY_MESSAGE_ID_REGEX = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\Z/.freeze - def address_formatted_correctly? - config.address && - config.address.include?("%{key}") + def enabled? + config.enabled && config.address end def reply_address(key) @@ -24,6 +21,13 @@ module Gitlab match[1] end + def key_from_fallback_reply_message_id(message_id) + match = message_id.match(FALLBACK_REPLY_MESSAGE_ID_REGEX) + return unless match + + match[1] + end + def config Gitlab.config.incoming_email end diff --git a/lib/tasks/gitlab/check.rake b/lib/tasks/gitlab/check.rake index 27ed57efe55..effb8eb6001 100644 --- a/lib/tasks/gitlab/check.rake +++ b/lib/tasks/gitlab/check.rake @@ -623,7 +623,6 @@ namespace :gitlab do start_checking "Reply by email" if Gitlab.config.incoming_email.enabled - check_address_formatted_correctly check_imap_authentication if Rails.env.production? @@ -643,20 +642,6 @@ namespace :gitlab do # Checks ######################## - def check_address_formatted_correctly - print "Address formatted correctly? ... " - - if Gitlab::IncomingEmail.address_formatted_correctly? - puts "yes".green - else - puts "no".red - try_fixing_it( - "Make sure that the address in config/gitlab.yml includes the '%{key}' placeholder." - ) - fix_and_rerun - end - end - def check_initd_configured_correctly print "Init.d configured correctly? ... " -- cgit v1.2.3 From 473b261261ab89fb921d12f05926ad9e9de5faee Mon Sep 17 00:00:00 2001 From: Zeger-Jan van de Weg Date: Fri, 25 Mar 2016 22:21:50 +0100 Subject: Back dating of issues when creating throught the API --- lib/api/issues.rb | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/api/issues.rb b/lib/api/issues.rb index e5ae88eb96f..1fee1dee1a6 100644 --- a/lib/api/issues.rb +++ b/lib/api/issues.rb @@ -111,17 +111,21 @@ module API # Create a new project issue # # Parameters: - # id (required) - The ID of a project - # title (required) - The title of an issue - # description (optional) - The description of an issue - # assignee_id (optional) - The ID of a user to assign issue + # id (required) - The ID of a project + # title (required) - The title of an issue + # description (optional) - The description of an issue + # assignee_id (optional) - The ID of a user to assign issue # milestone_id (optional) - The ID of a milestone to assign issue - # labels (optional) - The labels of an issue + # labels (optional) - The labels of an issue + # created_at (optional) - The date # Example Request: # POST /projects/:id/issues post ":id/issues" do required_attributes! [:title] - attrs = attributes_for_keys [:title, :description, :assignee_id, :milestone_id] + + keys = [:title, :description, :assignee_id, :milestone_id] + keys << :created_at if current_user.admin? || user_project.owner == current_user + attrs = attributes_for_keys(keys) # Validate label names in advance if (errors = validate_label_params(params)).any? -- cgit v1.2.3 From 4cd1b9f4d82efe3ffe810dabf6929a749c36c1bf Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 23 Mar 2016 11:24:18 +0100 Subject: Refactor builds badge, encapsulate inside a class --- lib/gitlab/badge/build.rb | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 lib/gitlab/badge/build.rb (limited to 'lib') diff --git a/lib/gitlab/badge/build.rb b/lib/gitlab/badge/build.rb new file mode 100644 index 00000000000..28a2391dbf8 --- /dev/null +++ b/lib/gitlab/badge/build.rb @@ -0,0 +1,24 @@ +module Gitlab + module Badge + ## + # Build badge + # + class Build + def initialize(project, ref) + @image = ::Ci::ImageForBuildService.new.execute(project, ref: ref) + end + + def to_s + @image[:name].sub(/\.svg$/, '') + end + + def type + 'image/svg+xml' + end + + def data + File.read(@image[:path]) + end + end + end +end -- cgit v1.2.3 From 701976e0815c273ff4a4c6e4d3489db0ce2f0860 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Thu, 24 Mar 2016 12:28:43 +0100 Subject: Add uploads rewriter and use it when moving issue --- lib/gitlab/gfm/uploads_rewriter.rb | 56 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 lib/gitlab/gfm/uploads_rewriter.rb (limited to 'lib') diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb new file mode 100644 index 00000000000..778b6fe9f9d --- /dev/null +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -0,0 +1,56 @@ +module Gitlab + module Gfm + ## + # Class that rewrites markdown links for uploads + # + # Using a pattern defined in `FileUploader` copies files to a new project + # and rewrites all links to uploads in ain a given text. + # + class UploadsRewriter + def initialize(text, source_project, _current_user) + @text = text + @source_project = source_project + @pattern = FileUploader::MARKDOWN_PATTERN + end + + def rewrite(target_project) + return unless @text + + new_uploader = file_uploader(target_project) + @text.gsub(@pattern) do |markdown_link| + old_file = find_file(@source_project, $~[:secret], $~[:file]) + return markdown_link unless old_file.exists? + + new_uploader.store!(old_file) + new_uploader.to_h[:markdown] + end + end + + def has_uploads? + !(@text =~ @pattern).nil? + end + + def files + referenced_files = @text.scan(@pattern).map do + find_file(@source_project, $~[:secret], $~[:file]) + end + + referenced_files.compact.select(&:exists?) + end + + private + + def find_file(project, secret, file) + uploader = file_uploader(project, secret) + uploader.retrieve_from_store!(file) + uploader.file + end + + def file_uploader(*args) + uploader = FileUploader.new(*args) + uploader.define_singleton_method(:move_to_store) { false } + uploader + end + end + end +end -- cgit v1.2.3 From f2674c7b98c69668093583e4590223b7040b5b33 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Tue, 29 Mar 2016 13:21:57 +0200 Subject: Refactor uploads rewriter used when moving issue --- lib/gitlab/gfm/reference_rewriter.rb | 9 ++++++--- lib/gitlab/gfm/uploads_rewriter.rb | 21 +++++++++++---------- 2 files changed, 17 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index a1c6ee7bd69..5f906d07177 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -34,16 +34,19 @@ module Gitlab @source_project = source_project @current_user = current_user @original_html = markdown(text) + @pattern = Gitlab::ReferenceExtractor.references_pattern end def rewrite(target_project) - pattern = Gitlab::ReferenceExtractor.references_pattern - - @text.gsub(pattern) do |reference| + @text.gsub(@pattern) do |reference| unfold_reference(reference, Regexp.last_match, target_project) end end + def needs_rewrite? + !(@text =~ @pattern).nil? + end + private def unfold_reference(reference, match, target_project) diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 778b6fe9f9d..5818766c974 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -3,8 +3,9 @@ module Gitlab ## # Class that rewrites markdown links for uploads # - # Using a pattern defined in `FileUploader` copies files to a new project - # and rewrites all links to uploads in ain a given text. + # Using a pattern defined in `FileUploader` it copies files to a new + # project and rewrites all links to uploads in in a given text. + # # class UploadsRewriter def initialize(text, source_project, _current_user) @@ -17,17 +18,17 @@ module Gitlab return unless @text new_uploader = file_uploader(target_project) - @text.gsub(@pattern) do |markdown_link| - old_file = find_file(@source_project, $~[:secret], $~[:file]) - return markdown_link unless old_file.exists? + @text.gsub(@pattern) do |markdown| + file = find_file(@source_project, $~[:secret], $~[:file]) + return markdown unless file.try(:exists?) - new_uploader.store!(old_file) + new_uploader.store!(file) new_uploader.to_h[:markdown] end end - def has_uploads? - !(@text =~ @pattern).nil? + def needs_rewrite? + files.any? end def files @@ -46,8 +47,8 @@ module Gitlab uploader.file end - def file_uploader(*args) - uploader = FileUploader.new(*args) + def file_uploader(project, secret = nil) + uploader = FileUploader.new(project, secret) uploader.define_singleton_method(:move_to_store) { false } uploader end -- cgit v1.2.3 From e64b1e52a23016e51d581b87c08beaa4b18da689 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 30 Mar 2016 10:42:39 +0200 Subject: Check if GFM rewriters need rewrite internally --- lib/gitlab/gfm/reference_rewriter.rb | 2 ++ lib/gitlab/gfm/uploads_rewriter.rb | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 5f906d07177..47e1aa67976 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -38,6 +38,8 @@ module Gitlab end def rewrite(target_project) + return @text unless needs_rewrite? + @text.gsub(@pattern) do |reference| unfold_reference(reference, Regexp.last_match, target_project) end diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 5818766c974..bdf054a6192 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -15,7 +15,7 @@ module Gitlab end def rewrite(target_project) - return unless @text + return @text unless needs_rewrite? new_uploader = file_uploader(target_project) @text.gsub(@pattern) do |markdown| -- cgit v1.2.3 From 99ee822857cf3fdf0a2ac91c0d13ea68c79e8ba8 Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 30 Mar 2016 10:56:25 +0200 Subject: Add method that returns markdown in file uploader --- lib/gitlab/gfm/reference_rewriter.rb | 2 +- lib/gitlab/gfm/uploads_rewriter.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 47e1aa67976..78d7a4f27cf 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -46,7 +46,7 @@ module Gitlab end def needs_rewrite? - !(@text =~ @pattern).nil? + @text =~ @pattern end private diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index bdf054a6192..2e61f799a03 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -23,7 +23,7 @@ module Gitlab return markdown unless file.try(:exists?) new_uploader.store!(file) - new_uploader.to_h[:markdown] + new_uploader.to_markdown end end -- cgit v1.2.3 From b9f57192853d100c90b1d46491838a98d5ae4bae Mon Sep 17 00:00:00 2001 From: Grzegorz Bizon Date: Wed, 30 Mar 2016 12:11:27 +0200 Subject: Remove reduntant `move_to_store` override --- lib/gitlab/gfm/uploads_rewriter.rb | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/gfm/uploads_rewriter.rb b/lib/gitlab/gfm/uploads_rewriter.rb index 2e61f799a03..abc8c8c55e6 100644 --- a/lib/gitlab/gfm/uploads_rewriter.rb +++ b/lib/gitlab/gfm/uploads_rewriter.rb @@ -17,11 +17,11 @@ module Gitlab def rewrite(target_project) return @text unless needs_rewrite? - new_uploader = file_uploader(target_project) @text.gsub(@pattern) do |markdown| file = find_file(@source_project, $~[:secret], $~[:file]) return markdown unless file.try(:exists?) + new_uploader = FileUploader.new(target_project) new_uploader.store!(file) new_uploader.to_markdown end @@ -42,16 +42,10 @@ module Gitlab private def find_file(project, secret, file) - uploader = file_uploader(project, secret) + uploader = FileUploader.new(project, secret) uploader.retrieve_from_store!(file) uploader.file end - - def file_uploader(project, secret = nil) - uploader = FileUploader.new(project, secret) - uploader.define_singleton_method(:move_to_store) { false } - uploader - end end end end -- cgit v1.2.3 From 2d544d5445d12e45aea1341ab96059ca377484c1 Mon Sep 17 00:00:00 2001 From: James Lopez Date: Wed, 30 Mar 2016 18:48:28 +0200 Subject: spec and fix for fogbugz lonely user problem --- lib/gitlab/fogbugz_import/client.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/gitlab/fogbugz_import/client.rb b/lib/gitlab/fogbugz_import/client.rb index 431d50882fd..2152182b37f 100644 --- a/lib/gitlab/fogbugz_import/client.rb +++ b/lib/gitlab/fogbugz_import/client.rb @@ -26,7 +26,7 @@ module Gitlab def user_map users = {} res = @api.command(:listPeople) - res['people']['person'].each do |user| + [res['people']['person']].flatten.each do |user| users[user['ixPerson']] = { name: user['sFullName'], email: user['sEmail'] } end users -- cgit v1.2.3 From 5830d80b8d2b87daa9123dc248ae00e2ed90069c Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 30 Mar 2016 19:29:17 +0200 Subject: Pre-calculate Emoji digests By pre-calculating the digests we can manually construct the emoji URLs, removing the need for using Rails' asset URL helpers. The reason we don't want to use these helpers for Emojis is two-fold: 1. Rails' image_url() method is slow, really slow. For one it _might_ have to calculate digests but it also performs a lot of other intensive operations (judging by the source code and based on measuring timings). 2. We have a lot of Emoji which coupled with the above can result in it taking minutes to load Emoji autocomplete data. Using this pre-calculation setup generating the digests takes around 7 seconds (including the time it takes to start Rails/Rake), and only around 600 milliseconds to load _all_ the autocomplete data of a project (measured locally). This commit _does_ change the Emoji URLs from absolute to relative URLs as these are much easier to generate. To update the Emoji data simply run: rake gemojione:digests Then commit any changes. Fixes gitlab-org/gitlab-ce#14009 --- lib/award_emoji.rb | 19 +++++++++++++++++++ lib/tasks/gemojione.rake | 48 ++++++++++++++++++++++++++++++++++-------------- 2 files changed, 53 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/award_emoji.rb b/lib/award_emoji.rb index 783fcfb61ad..4fc3443ac68 100644 --- a/lib/award_emoji.rb +++ b/lib/award_emoji.rb @@ -48,4 +48,23 @@ class AwardEmoji JSON.parse(File.read(json_path)) end end + + # Returns an Array of Emoji names and their asset URLs. + def self.urls + @urls ||= begin + path = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') + prefix = Gitlab::Application.config.assets.prefix + digest = Gitlab::Application.config.assets.digest + + JSON.parse(File.read(path)).map do |hash| + if digest + fname = "#{hash['unicode']}-#{hash['digest']}" + else + fname = hash['unicode'] + end + + { name: hash['name'], path: "#{prefix}/#{fname}.png" } + end + end + end end diff --git a/lib/tasks/gemojione.rake b/lib/tasks/gemojione.rake index cfaf4a129b1..7ec00a898fd 100644 --- a/lib/tasks/gemojione.rake +++ b/lib/tasks/gemojione.rake @@ -1,19 +1,39 @@ -# This task will generate a standard and Retina sprite of all of the current -# Gemojione Emojis, with the accompanying SCSS map. -# -# It will not appear in `rake -T` output, and the dependent gems are not -# included in the Gemfile by default, because this task will only be needed -# occasionally, such as when new Emojis are added to Gemojione. - -begin - require 'sprite_factory' - require 'rmagick' -rescue LoadError - # noop -end - namespace :gemojione do + desc 'Generates Emoji SHA256 digests' + task digests: :environment do + require 'digest/sha2' + require 'json' + + dir = Gemojione.index.images_path + + digests = AwardEmoji.emojis.map do |name, emoji_hash| + fpath = File.join(dir, "#{emoji_hash['unicode']}.png") + digest = Digest::SHA256.file(fpath).hexdigest + + { name: name, unicode: emoji_hash['unicode'], digest: digest } + end + + out = File.join(Rails.root, 'fixtures', 'emojis', 'digests.json') + + File.open(out, 'w') do |handle| + handle.write(JSON.pretty_generate(digests)) + end + end + + # This task will generate a standard and Retina sprite of all of the current + # Gemojione Emojis, with the accompanying SCSS map. + # + # It will not appear in `rake -T` output, and the dependent gems are not + # included in the Gemfile by default, because this task will only be needed + # occasionally, such as when new Emojis are added to Gemojione. task sprite: :environment do + begin + require 'sprite_factory' + require 'rmagick' + rescue LoadError + # noop + end + check_requirements! SIZE = 20 -- cgit v1.2.3 From 091b8a6ede2515bb555ec8662b9d933d70bda3e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 31 Mar 2016 09:20:27 +0200 Subject: Rename Note#for_project_snippet? to #for_snippet? MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- lib/gitlab/note_data_builder.rb | 2 +- lib/gitlab/url_builder.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/note_data_builder.rb b/lib/gitlab/note_data_builder.rb index 71cf6a0d886..18523e0aefe 100644 --- a/lib/gitlab/note_data_builder.rb +++ b/lib/gitlab/note_data_builder.rb @@ -41,7 +41,7 @@ module Gitlab data[:issue] = note.noteable.hook_attrs elsif note.for_merge_request? data[:merge_request] = note.noteable.hook_attrs - elsif note.for_project_snippet? + elsif note.for_snippet? data[:snippet] = note.noteable.hook_attrs end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 6f0d02cafd1..7486510a4af 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -46,7 +46,7 @@ module Gitlab merge_request = MergeRequest.find(note.noteable_id) merge_request_url(merge_request, anchor: "note_#{note.id}") - elsif note.for_project_snippet? + elsif note.for_snippet? snippet = Snippet.find(note.noteable_id) project_snippet_url(snippet, anchor: "note_#{note.id}") -- cgit v1.2.3 From e60f034126712b7e5a3b3ff9c5e92359aaf96e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Thu, 31 Mar 2016 09:21:20 +0200 Subject: Fix view of notes in search results when noteable is a snippet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Also, streamline the view. Signed-off-by: Rémy Coutable --- lib/gitlab/url_builder.rb | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 7486510a4af..e157bb96f2a 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -2,6 +2,7 @@ module Gitlab class UrlBuilder include Gitlab::Application.routes.url_helpers include GitlabRoutingHelper + include ActionView::RecordIdentifier def initialize(type) @type = type @@ -37,19 +38,16 @@ module Gitlab namespace_project_commit_url(namespace_id: note.project.namespace, id: note.commit_id, project_id: note.project, - anchor: "note_#{note.id}") + anchor: dom_id(note)) elsif note.for_issue? issue = Issue.find(note.noteable_id) - issue_url(issue, - anchor: "note_#{note.id}") + issue_url(issue, anchor: dom_id(note)) elsif note.for_merge_request? merge_request = MergeRequest.find(note.noteable_id) - merge_request_url(merge_request, - anchor: "note_#{note.id}") + merge_request_url(merge_request, anchor: dom_id(note)) elsif note.for_snippet? snippet = Snippet.find(note.noteable_id) - project_snippet_url(snippet, - anchor: "note_#{note.id}") + project_snippet_url(snippet, anchor: dom_id(note)) end end end -- cgit v1.2.3 From 85cc1729596ac1e5b31d8cfa1daa07477db6033d Mon Sep 17 00:00:00 2001 From: connorshea Date: Thu, 31 Mar 2016 16:40:39 -0600 Subject: Remove "Congratulations!" tweet button on newly-created project. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I’ve removed everything related to the feature based on this commit: ce08f919f34fd8849834365 Resolves #10857. --- lib/api/entities.rb | 1 - lib/gitlab/current_settings.rb | 1 - 2 files changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f686c568bee..b7de575cdcd 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -334,7 +334,6 @@ module API expose :updated_at expose :home_page_url expose :default_branch_protection - expose :twitter_sharing_enabled expose :restricted_visibility_levels expose :max_attachment_size expose :session_expire_delay diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb index 761b63e98f6..1acc22fe5bf 100644 --- a/lib/gitlab/current_settings.rb +++ b/lib/gitlab/current_settings.rb @@ -21,7 +21,6 @@ module Gitlab default_branch_protection: Settings.gitlab['default_branch_protection'], signup_enabled: Settings.gitlab['signup_enabled'], signin_enabled: Settings.gitlab['signin_enabled'], - twitter_sharing_enabled: Settings.gitlab['twitter_sharing_enabled'], gravatar_enabled: Settings.gravatar['enabled'], sign_in_text: Settings.extra['sign_in_text'], restricted_visibility_levels: Settings.gitlab['restricted_visibility_levels'], -- cgit v1.2.3 From 84b0ab77667b85a42db8a5a02d9758657af66f16 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 24 Mar 2016 17:00:26 +0100 Subject: Added & use Gitlab::Routing for URL helpers Rails' "url_helpers" method creates an anonymous Module (which a bunch of methods) on every call. By caching the output of this method in a dedicated method we can shave off about 10 seconds of loading time for an issue with around 200 comments. --- lib/api/entities.rb | 4 ++-- lib/banzai/filter/commit_range_reference_filter.rb | 2 +- lib/banzai/filter/commit_reference_filter.rb | 2 +- lib/banzai/filter/label_reference_filter.rb | 2 +- lib/banzai/filter/merge_request_reference_filter.rb | 2 +- lib/banzai/filter/milestone_reference_filter.rb | 2 +- lib/banzai/filter/snippet_reference_filter.rb | 2 +- lib/banzai/filter/user_reference_filter.rb | 2 +- lib/gitlab/email/message/repository_push.rb | 2 +- lib/gitlab/routing.rb | 13 +++++++++++++ lib/gitlab/url_builder.rb | 2 +- 11 files changed, 24 insertions(+), 11 deletions(-) create mode 100644 lib/gitlab/routing.rb (limited to 'lib') diff --git a/lib/api/entities.rb b/lib/api/entities.rb index f686c568bee..c452aed27df 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -8,7 +8,7 @@ module API expose :id, :state, :avatar_url expose :web_url do |user, options| - Gitlab::Application.routes.url_helpers.user_url(user) + Gitlab::Routing.url_helpers.user_url(user) end end @@ -89,7 +89,7 @@ module API expose :avatar_url expose :web_url do |group, options| - Gitlab::Application.routes.url_helpers.group_url(group) + Gitlab::Routing.url_helpers.group_url(group) end end diff --git a/lib/banzai/filter/commit_range_reference_filter.rb b/lib/banzai/filter/commit_range_reference_filter.rb index 470727ee312..b469ea0f626 100644 --- a/lib/banzai/filter/commit_range_reference_filter.rb +++ b/lib/banzai/filter/commit_range_reference_filter.rb @@ -43,7 +43,7 @@ module Banzai end def url_for_object(range, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_compare_url(project.namespace, project, range.to_param.merge(only_path: context[:only_path])) end diff --git a/lib/banzai/filter/commit_reference_filter.rb b/lib/banzai/filter/commit_reference_filter.rb index 713a56ba949..bd88207326c 100644 --- a/lib/banzai/filter/commit_reference_filter.rb +++ b/lib/banzai/filter/commit_reference_filter.rb @@ -37,7 +37,7 @@ module Banzai end def url_for_object(commit, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_commit_url(project.namespace, project, commit, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/label_reference_filter.rb b/lib/banzai/filter/label_reference_filter.rb index 8147e5ed3c7..a2987850d03 100644 --- a/lib/banzai/filter/label_reference_filter.rb +++ b/lib/banzai/filter/label_reference_filter.rb @@ -31,7 +31,7 @@ module Banzai end def url_for_object(label, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_issues_url(project.namespace, project, label_name: label.name, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/merge_request_reference_filter.rb b/lib/banzai/filter/merge_request_reference_filter.rb index 57c71708992..cad38a51851 100644 --- a/lib/banzai/filter/merge_request_reference_filter.rb +++ b/lib/banzai/filter/merge_request_reference_filter.rb @@ -14,7 +14,7 @@ module Banzai end def url_for_object(mr, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_merge_request_url(project.namespace, project, mr, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/milestone_reference_filter.rb b/lib/banzai/filter/milestone_reference_filter.rb index 8f710a92bdc..4cb82178024 100644 --- a/lib/banzai/filter/milestone_reference_filter.rb +++ b/lib/banzai/filter/milestone_reference_filter.rb @@ -11,7 +11,7 @@ module Banzai end def url_for_object(issue, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_milestone_url(project.namespace, project, milestone, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/snippet_reference_filter.rb b/lib/banzai/filter/snippet_reference_filter.rb index c870a42f741..d507eb5ebe1 100644 --- a/lib/banzai/filter/snippet_reference_filter.rb +++ b/lib/banzai/filter/snippet_reference_filter.rb @@ -14,7 +14,7 @@ module Banzai end def url_for_object(snippet, project) - h = Gitlab::Application.routes.url_helpers + h = Gitlab::Routing.url_helpers h.namespace_project_snippet_url(project.namespace, project, snippet, only_path: context[:only_path]) end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 24f16f8b547..989fa64e078 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -90,7 +90,7 @@ module Banzai private def urls - Gitlab::Application.routes.url_helpers + Gitlab::Routing.url_helpers end def link_class diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index 41f0edcaf7e..8f9be6cd9a3 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -5,7 +5,7 @@ module Gitlab attr_accessor :recipient attr_reader :author_id, :ref, :action - include Gitlab::Application.routes.url_helpers + include Gitlab::Routing.url_helpers delegate :namespace, :name_with_namespace, to: :project, prefix: :project delegate :name, to: :author, prefix: :author diff --git a/lib/gitlab/routing.rb b/lib/gitlab/routing.rb new file mode 100644 index 00000000000..5132177de51 --- /dev/null +++ b/lib/gitlab/routing.rb @@ -0,0 +1,13 @@ +module Gitlab + module Routing + # Returns the URL helpers Module. + # + # This method caches the output as Rails' "url_helpers" method creates an + # anonymous module every time it's called. + # + # Returns a Module. + def self.url_helpers + @url_helpers ||= Gitlab::Application.routes.url_helpers + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 6f0d02cafd1..22c91be9207 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -1,6 +1,6 @@ module Gitlab class UrlBuilder - include Gitlab::Application.routes.url_helpers + include Gitlab::Routing.url_helpers include GitlabRoutingHelper def initialize(type) -- cgit v1.2.3 From 57bde0ce65caf2cbbb6a57f21435639cdaa06225 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 24 Mar 2016 15:53:38 +0100 Subject: Cache Banzai projects/objects using RequestStore This was originally suggested by @ayufan and modified to be a bit cleaner and use RequestStore instead of a regular Hash. By caching the output of the two methods involved the number of queries is reduced significantly. For example, for an issue with 200 notes (of which 100 reference a number of merge requests) this cuts down the amount of queries from around 6300 to around 3300. --- lib/banzai/filter/abstract_reference_filter.rb | 71 ++++++++++++++++++++++++-- 1 file changed, 67 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 34c38913474..41fd4be76ac 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -62,11 +62,53 @@ module Banzai # Example: project.merge_requests.find end + def find_object_cached(project, id) + if RequestStore.active? + cache = find_objects_cache[object_class][project.id] + + if cache.key?(id) + cache[id] + else + cache[id] = find_object(project, id) + end + else + find_object(project, id) + end + end + + def project_from_ref_cache(ref) + if RequestStore.active? + cache = project_refs_cache + + if cache.key?(ref) + cache[ref] + else + cache[ref] = project_from_ref(ref) + end + else + project_from_ref(ref) + end + end + def url_for_object(object, project) # Implement in child class # Example: project_merge_request_url end + def url_for_object_cached(object, project) + if RequestStore.active? + cache = url_for_object_cache[object_class][project.id] + + if cache.key?(object) + cache[object] + else + cache[object] = url_for_object(object, project) + end + else + url_for_object(object, project) + end + end + def call if object_class.reference_pattern # `#123` @@ -109,9 +151,9 @@ module Banzai # have `gfm` and `gfm-OBJECT_NAME` class names attached for styling. def object_link_filter(text, pattern, link_text: nil) references_in(text, pattern) do |match, id, project_ref, matches| - project = project_from_ref(project_ref) + project = project_from_ref_cache(project_ref) - if project && object = find_object(project, id) + if project && object = find_object_cached(project, id) title = object_link_title(object) klass = reference_class(object_sym) @@ -121,8 +163,11 @@ module Banzai object_sym => object.id ) - url = matches[:url] if matches.names.include?("url") - url ||= url_for_object(object, project) + if matches.names.include?("url") && matches[:url] + url = matches[:url] + else + url = url_for_object_cached(object, project) + end text = link_text || object_link_text(object, matches) @@ -157,6 +202,24 @@ module Banzai text end + + private + + def project_refs_cache + RequestStore[:banzai_project_refs] ||= {} + end + + def find_objects_cache + RequestStore[:banzai_find_objects_cache] ||= Hash.new do |hash, key| + hash[key] = Hash.new { |h, k| h[k] = {} } + end + end + + def url_for_object_cache + RequestStore[:banzai_url_for_object] ||= Hash.new do |hash, key| + hash[key] = Hash.new { |h, k| h[k] = {} } + end + end end end end -- cgit v1.2.3 From 141148057703048f5c409c040c80c277f7747273 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 24 Mar 2016 15:57:22 +0100 Subject: Refactor processing of various Banzai filters These filters now use a single iteration over all the document nodes instead of multiple ones. This in turn allows variables to be re-used (e.g. links only have to be unescaped once). Combined with some other refactoring this can drastically reduce render timings. --- lib/banzai/filter/abstract_reference_filter.rb | 57 +++++--- .../filter/external_issue_reference_filter.rb | 24 +++- lib/banzai/filter/reference_filter.rb | 148 +++++++-------------- lib/banzai/filter/user_reference_filter.rb | 25 +++- 4 files changed, 120 insertions(+), 134 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 41fd4be76ac..16b703a3323 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -110,30 +110,45 @@ module Banzai end def call - if object_class.reference_pattern - # `#123` - replace_text_nodes_matching(object_class.reference_pattern) do |content| - object_link_filter(content, object_class.reference_pattern) - end + return doc if project.nil? - # `[Issue](#123)`, which is turned into - # `Issue` - replace_link_nodes_with_href(object_class.reference_pattern) do |link, text| - object_link_filter(link, object_class.reference_pattern, link_text: text) - end - end + ref_pattern = object_class.reference_pattern + link_pattern = object_class.link_reference_pattern - if object_class.link_reference_pattern - # `http://gitlab.example.com/namespace/project/issues/123`, which is turned into - # `http://gitlab.example.com/namespace/project/issues/123` - replace_link_nodes_with_text(object_class.link_reference_pattern) do |text| - object_link_filter(text, object_class.link_reference_pattern) - end + each_node do |node| + if text_node?(node) && ref_pattern + replace_text_when_pattern_matches(node, ref_pattern) do |content| + object_link_filter(content, ref_pattern) + end + + elsif element_node?(node) + yield_valid_link(node) do |link, text| + if ref_pattern && link =~ /\A#{ref_pattern}/ + replace_link_node_with_href(node, link) do + object_link_filter(link, ref_pattern, link_text: text) + end - # `[Issue](http://gitlab.example.com/namespace/project/issues/123)`, which is turned into - # `Issue` - replace_link_nodes_with_href(object_class.link_reference_pattern) do |link, text| - object_link_filter(link, object_class.link_reference_pattern, link_text: text) + next + end + + next unless link_pattern + + if link == text && text =~ /\A#{link_pattern}/ + replace_link_node_with_text(node, link) do + object_link_filter(text, link_pattern) + end + + next + end + + if link =~ /\A#{link_pattern}\z/ + replace_link_node_with_href(node, link) do + object_link_filter(link, link_pattern, link_text: text) + end + + next + end + end end end diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index edc26386903..a3b66c4645a 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -37,13 +37,27 @@ module Banzai # Early return if the project isn't using an external tracker return doc if project.nil? || project.default_issues_tracker? - replace_text_nodes_matching(ExternalIssue.reference_pattern) do |content| - issue_link_filter(content) - end + ref_pattern = ExternalIssue.reference_pattern + ref_start_pattern = /\A#{ref_pattern}\z/ + + each_node do |node| + if text_node?(node) + replace_text_when_pattern_matches(node, ref_pattern) do |content| + issue_link_filter(content) + end - replace_link_nodes_with_href(ExternalIssue.reference_pattern) do |link, text| - issue_link_filter(link, link_text: text) + elsif element_node?(node) + yield_valid_link(node) do |link, text| + if link =~ ref_start_pattern + replace_link_node_with_href(node, link) do + issue_link_filter(link, link_text: text) + end + end + end + end end + + doc end # Replace `JIRA-123` issue references in text with links to the referenced diff --git a/lib/banzai/filter/reference_filter.rb b/lib/banzai/filter/reference_filter.rb index a3326ae042c..31386cf851c 100644 --- a/lib/banzai/filter/reference_filter.rb +++ b/lib/banzai/filter/reference_filter.rb @@ -52,18 +52,13 @@ module Banzai html.html_safe? ? html : ERB::Util.html_escape_once(html) end - def ignore_parents - @ignore_parents ||= begin - # Don't look for references in text nodes that are children of these - # elements. + def ignore_ancestor_query + @ignore_ancestor_query ||= begin parents = %w(pre code a style) parents << 'blockquote' if context[:ignore_blockquotes] - parents.to_set - end - end - def ignored_ancestry?(node) - has_ancestor?(node, ignore_parents) + parents.map { |n| "ancestor::#{n}" }.join(' or ') + end end def project @@ -74,119 +69,66 @@ module Banzai "gfm gfm-#{type}" end - # Iterate through the document's text nodes, yielding the current node's - # content if: - # - # * The `project` context value is present AND - # * The node's content matches `pattern` AND - # * The node is not an ancestor of an ignored node type - # - # pattern - Regex pattern against which to match the node's content - # - # Yields the current node's String contents. The result of the block will - # replace the node's existing content and update the current document. + # Ensure that a :project key exists in context # - # Returns the updated Nokogiri::HTML::DocumentFragment object. - def replace_text_nodes_matching(pattern) - return doc if project.nil? - - search_text_nodes(doc).each do |node| - next if ignored_ancestry?(node) - next unless node.text =~ pattern - - content = node.to_html - - html = yield content - - next if html == content - - node.replace(html) - end - - doc + # Note that while the key might exist, its value could be nil! + def validate + needs :project end - # Iterate through the document's link nodes, yielding the current node's - # content if: - # - # * The `project` context value is present AND - # * The node's content matches `pattern` - # - # pattern - Regex pattern against which to match the node's content - # - # Yields the current node's String contents. The result of the block will - # replace the node and update the current document. + # Iterates over all and text() nodes in a document. # - # Returns the updated Nokogiri::HTML::DocumentFragment object. - def replace_link_nodes_with_text(pattern) - return doc if project.nil? + # Nodes are skipped whenever their ancestor is one of the nodes returned + # by `ignore_ancestor_query`. Link tags are not processed if they have a + # "gfm" class or the "href" attribute is empty. + def each_node + query = %Q{descendant-or-self::text()[not(#{ignore_ancestor_query})] + | descendant-or-self::a[ + not(contains(concat(" ", @class, " "), " gfm ")) and not(@href = "") + ]} - doc.xpath('descendant-or-self::a').each do |node| - klass = node.attr('class') - next if klass && klass.include?('gfm') - - link = node.attr('href') - text = node.text - - next unless link && text - - link = CGI.unescape(link) - next unless link.force_encoding('UTF-8').valid_encoding? - # Ignore ending punctionation like periods or commas - next unless link == text && text =~ /\A#{pattern}/ - - html = yield text + doc.xpath(query).each do |node| + yield node + end + end - next if html == text + # Yields the link's URL and text whenever the node is a valid tag. + def yield_valid_link(node) + link = CGI.unescape(node.attr('href').to_s) + text = node.text - node.replace(html) - end + return unless link.force_encoding('UTF-8').valid_encoding? - doc + yield link, text end - # Iterate through the document's link nodes, yielding the current node's - # content if: - # - # * The `project` context value is present AND - # * The node's HREF matches `pattern` - # - # pattern - Regex pattern against which to match the node's HREF - # - # Yields the current node's String HREF and String content. - # The result of the block will replace the node and update the current document. - # - # Returns the updated Nokogiri::HTML::DocumentFragment object. - def replace_link_nodes_with_href(pattern) - return doc if project.nil? + def replace_text_when_pattern_matches(node, pattern) + return unless node.text =~ pattern - doc.xpath('descendant-or-self::a').each do |node| - klass = node.attr('class') - next if klass && klass.include?('gfm') + content = node.to_html + html = yield content - link = node.attr('href') - text = node.text + node.replace(html) unless content == html + end - next unless link && text - link = CGI.unescape(link) - next unless link.force_encoding('UTF-8').valid_encoding? - next unless link && link =~ /\A#{pattern}\z/ + def replace_link_node_with_text(node, link) + html = yield - html = yield link, text + node.replace(html) unless html == node.text + end - next if html == link + def replace_link_node_with_href(node, link) + html = yield - node.replace(html) - end + node.replace(html) unless html == link + end - doc + def text_node?(node) + node.is_a?(Nokogiri::XML::Text) end - # Ensure that a :project key exists in context - # - # Note that while the key might exist, its value could be nil! - def validate - needs :project + def element_node?(node) + node.is_a?(Nokogiri::XML::Element) end end end diff --git a/lib/banzai/filter/user_reference_filter.rb b/lib/banzai/filter/user_reference_filter.rb index 989fa64e078..eea3af842b6 100644 --- a/lib/banzai/filter/user_reference_filter.rb +++ b/lib/banzai/filter/user_reference_filter.rb @@ -59,13 +59,28 @@ module Banzai end def call - replace_text_nodes_matching(User.reference_pattern) do |content| - user_link_filter(content) + return doc if project.nil? + + ref_pattern = User.reference_pattern + ref_pattern_start = /\A#{ref_pattern}\z/ + + each_node do |node| + if text_node?(node) + replace_text_when_pattern_matches(node, ref_pattern) do |content| + user_link_filter(content) + end + elsif element_node?(node) + yield_valid_link(node) do |link, text| + if link =~ ref_pattern_start + replace_link_node_with_href(node, link) do + user_link_filter(link, link_text: text) + end + end + end + end end - replace_link_nodes_with_href(User.reference_pattern) do |link, text| - user_link_filter(link, link_text: text) - end + doc end # Replace `@user` user references in text with links to the referenced -- cgit v1.2.3 From 8c49eaa937ed3d4332c54e8b0929c328a85d7fe4 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Thu, 24 Mar 2016 16:27:52 +0100 Subject: Cache Banzai class methods returning static data These methods always return the same data for every class so there's no point in computing their values on every call. --- lib/banzai/filter/abstract_reference_filter.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 16b703a3323..051b94f9ce1 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -11,15 +11,15 @@ module Banzai end def self.object_name - object_class.name.underscore + @object_name ||= object_class.name.underscore end def self.object_sym - object_name.to_sym + @object_sym ||= object_name.to_sym end def self.data_reference - "data-#{object_name.dasherize}" + @data_reference ||= "data-#{object_name.dasherize}" end # Public: Find references in text (like `!123` for merge requests) -- cgit v1.2.3 From 9fa94326dba11ca3b9197a4f084ba2883c29bdff Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Tue, 29 Mar 2016 11:58:05 +0200 Subject: Memoize object class titles For an issue with around 200 notes this cuts down timings by around 150 milliseconds. --- lib/banzai/filter/abstract_reference_filter.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 051b94f9ce1..02ef27cf577 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -22,6 +22,10 @@ module Banzai @data_reference ||= "data-#{object_name.dasherize}" end + def self.object_class_title + @object_title ||= object_class.name.titleize + end + # Public: Find references in text (like `!123` for merge requests) # # AnyReferenceFilter.references_in(text) do |match, id, project_ref, matches| @@ -53,6 +57,10 @@ module Banzai self.class.object_sym end + def object_class_title + self.class.object_class_title + end + def references_in(*args, &block) self.class.references_in(*args, &block) end @@ -206,7 +214,7 @@ module Banzai end def object_link_title(object) - "#{object_class.name.titleize}: #{object.title}" + "#{object_class_title}: #{object.title}" end def object_link_text(object, matches) -- cgit v1.2.3 From 7f0fd73eeb983782bac26bb983ec8ea52194b80a Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Wed, 30 Mar 2016 14:04:28 +0200 Subject: Cache default_issues_tracker? in Banzai Every object processed by ExternalIssueReferenceFilter can return a different Project instance when calling "project". For example, every note processed will have it's own associated Project. If we were to cache Project#default_issues_tracker? on Project level this would have no impact on Markdown rendering timings as the cache would have to be built for every Project instance without it ever being re-used. To work around this we cache Project#default_issues_tracker? in Banzai::Filter::ExternalIssueReferenceFilter using the project's _id_ instead of the whole object. This setup allows re-using of the cached data even when the Project instances used are different, as long as the actual project IDs are the same. --- lib/banzai/filter/external_issue_reference_filter.rb | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/banzai/filter/external_issue_reference_filter.rb b/lib/banzai/filter/external_issue_reference_filter.rb index a3b66c4645a..37344b90576 100644 --- a/lib/banzai/filter/external_issue_reference_filter.rb +++ b/lib/banzai/filter/external_issue_reference_filter.rb @@ -35,7 +35,7 @@ module Banzai def call # Early return if the project isn't using an external tracker - return doc if project.nil? || project.default_issues_tracker? + return doc if project.nil? || default_issues_tracker? ref_pattern = ExternalIssue.reference_pattern ref_start_pattern = /\A#{ref_pattern}\z/ @@ -90,6 +90,21 @@ module Banzai def url_for_issue(*args) IssuesHelper.url_for_issue(*args) end + + def default_issues_tracker? + if RequestStore.active? + default_issues_tracker_cache[project.id] ||= + project.default_issues_tracker? + else + project.default_issues_tracker? + end + end + + private + + def default_issues_tracker_cache + RequestStore[:banzai_default_issues_tracker_cache] ||= {} + end end end end -- cgit v1.2.3 From 915fd3f910921bc97abdda5f2ae63093c11533fd Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 4 Apr 2016 11:39:11 +0200 Subject: Cleaned up caching in AbstractReferenceFilter Cleaning this up any further is a bit tricky as the caches in question should only be evaluated if RequestStore is actually enabled. --- lib/banzai/filter/abstract_reference_filter.rb | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/banzai/filter/abstract_reference_filter.rb b/lib/banzai/filter/abstract_reference_filter.rb index 02ef27cf577..f21dbef216c 100644 --- a/lib/banzai/filter/abstract_reference_filter.rb +++ b/lib/banzai/filter/abstract_reference_filter.rb @@ -74,11 +74,7 @@ module Banzai if RequestStore.active? cache = find_objects_cache[object_class][project.id] - if cache.key?(id) - cache[id] - else - cache[id] = find_object(project, id) - end + get_or_set_cache(cache, id) { find_object(project, id) } else find_object(project, id) end @@ -88,11 +84,7 @@ module Banzai if RequestStore.active? cache = project_refs_cache - if cache.key?(ref) - cache[ref] - else - cache[ref] = project_from_ref(ref) - end + get_or_set_cache(cache, ref) { project_from_ref(ref) } else project_from_ref(ref) end @@ -107,11 +99,7 @@ module Banzai if RequestStore.active? cache = url_for_object_cache[object_class][project.id] - if cache.key?(object) - cache[object] - else - cache[object] = url_for_object(object, project) - end + get_or_set_cache(cache, object) { url_for_object(object, project) } else url_for_object(object, project) end @@ -243,6 +231,14 @@ module Banzai hash[key] = Hash.new { |h, k| h[k] = {} } end end + + def get_or_set_cache(cache, key) + if cache.key?(key) + cache[key] + else + cache[key] = yield + end + end end end end -- cgit v1.2.3