diff options
| -rw-r--r-- | app/workers/email_receiver_worker.rb | 16 | ||||
| -rw-r--r-- | lib/gitlab/email/attachment_uploader.rb | 35 | ||||
| -rw-r--r-- | lib/gitlab/email/html_cleaner.rb | 135 | ||||
| -rw-r--r-- | lib/gitlab/email/receiver.rb | 101 | ||||
| -rw-r--r-- | lib/gitlab/email/reply_parser.rb | 91 | ||||
| -rw-r--r-- | lib/gitlab/email_html_cleaner.rb | 133 | ||||
| -rw-r--r-- | lib/gitlab/email_receiver.rb | 192 | ||||
| -rw-r--r-- | spec/lib/gitlab/email/reply_parser_spec.rb | 191 | ||||
| -rw-r--r-- | spec/lib/gitlab/email_receiver_spec.rb | 481 |
9 files changed, 561 insertions, 814 deletions
diff --git a/app/workers/email_receiver_worker.rb b/app/workers/email_receiver_worker.rb index 2cfd64cefad..4f3556bfc03 100644 --- a/app/workers/email_receiver_worker.rb +++ b/app/workers/email_receiver_worker.rb @@ -7,7 +7,7 @@ class EmailReceiverWorker return unless Gitlab::ReplyByEmail.enabled? begin - Gitlab::EmailReceiver.new(raw).execute + Gitlab::Email::Receiver.new(raw).execute rescue => e handle_failure(raw, e) end @@ -22,20 +22,20 @@ class EmailReceiverWorker reason = nil case e - when Gitlab::EmailReceiver::SentNotificationNotFound + when Gitlab::Email::Receiver::SentNotificationNotFound reason = "We couldn't figure out what the email is in reply to. Please create your comment through the web interface." - when Gitlab::EmailReceiver::EmptyEmailError + when Gitlab::Email::Receiver::EmptyEmailError can_retry = true reason = "It appears that the email is blank. Make sure your reply is at the top of the email, we can't process inline replies." - when Gitlab::EmailReceiver::AutoGeneratedEmailError + when Gitlab::Email::Receiver::AutoGeneratedEmailError reason = "The email was marked as 'auto generated', which we can't accept. Please create your comment through the web interface." - when Gitlab::EmailReceiver::UserNotFoundError + when Gitlab::Email::Receiver::UserNotFoundError reason = "We couldn't figure out what user corresponds to the email. Please create your comment through the web interface." - when Gitlab::EmailReceiver::UserNotAuthorizedError + when Gitlab::Email::Receiver::UserNotAuthorizedError reason = "You are not allowed to respond to the thread you are replying to. If you believe this is in error, contact a staff member." - when Gitlab::EmailReceiver::NoteableNotFoundError + when Gitlab::Email::Receiver::NoteableNotFoundError reason = "The thread you are replying to no longer exists, perhaps it was deleted? If you believe this is in error, contact a staff member." - when Gitlab::EmailReceiver::InvalidNote + when Gitlab::Email::Receiver::InvalidNote can_retry = true reason = e.message else diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb new file mode 100644 index 00000000000..0c0f50f2751 --- /dev/null +++ b/lib/gitlab/email/attachment_uploader.rb @@ -0,0 +1,35 @@ +module Gitlab + module Email + module AttachmentUploader + attr_accessor :message + + def initialize(message) + @message = message + end + + def execute(project) + attachments = [] + + message.attachments.each do |attachment| + tmp = Tempfile.new("gitlab-email-attachment") + begin + File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded } + + file = { + tempfile: tmp, + filename: attachment.filename, + content_type: attachment.content_type + } + + link = ::Projects::UploadService.new(project, file).execute + attachments << link if link + ensure + tmp.close! + end + end + + attachments + end + end + end +end diff --git a/lib/gitlab/email/html_cleaner.rb b/lib/gitlab/email/html_cleaner.rb new file mode 100644 index 00000000000..e1ae9eee56c --- /dev/null +++ b/lib/gitlab/email/html_cleaner.rb @@ -0,0 +1,135 @@ +# Taken mostly from Discourse's Email::HtmlCleaner +module Gitlab + module Email + # HtmlCleaner cleans up the extremely dirty HTML that many email clients + # generate by stripping out any excess divs or spans, removing styling in + # the process (which also makes the html more suitable to be parsed as + # Markdown). + class HtmlCleaner + # Elements to hoist all children out of + HTML_HOIST_ELEMENTS = %w(div span font table tbody th tr td) + # Node types to always delete + HTML_DELETE_ELEMENT_TYPES = [ + Nokogiri::XML::Node::DTD_NODE, + Nokogiri::XML::Node::COMMENT_NODE, + ] + + # Private variables: + # @doc - nokogiri document + # @out - same as @doc, but only if trimming has occured + def initialize(html) + if html.is_a?(String) + @doc = Nokogiri::HTML(html) + else + @doc = html + end + end + + class << self + # HtmlCleaner.trim(inp, opts={}) + # + # Arguments: + # inp - Either a HTML string or a Nokogiri document. + # Options: + # :return => :doc, :string + # Specify the desired return type. + # Defaults to the type of the input. + # A value of :string is equivalent to calling get_document_text() + # on the returned document. + def trim(inp, opts={}) + cleaner = HtmlCleaner.new(inp) + + opts[:return] ||= (inp.is_a?(String) ? :string : :doc) + + if opts[:return] == :string + cleaner.output_html + else + cleaner.output_document + end + end + + # HtmlCleaner.get_document_text(doc) + # + # Get the body portion of the document, including html, as a string. + def get_document_text(doc) + body = doc.xpath('//body') + if body + body.inner_html + else + doc.inner_html + end + end + end + + def output_document + @out ||= begin + doc = @doc + trim_process_node doc + add_newlines doc + doc + end + end + + def output_html + HtmlCleaner.get_document_text(output_document) + end + + private + + def add_newlines(doc) + # Replace <br> tags with a markdown \n + doc.xpath('//br').each do |br| + br.replace(new_linebreak_node doc, 2) + end + # Surround <p> tags with newlines, to help with line-wise postprocessing + # and ensure markdown paragraphs + doc.xpath('//p').each do |p| + p.before(new_linebreak_node doc) + p.after(new_linebreak_node doc, 2) + end + end + + def new_linebreak_node(doc, count=1) + Nokogiri::XML::Text.new("\n" * count, doc) + end + + def trim_process_node(node) + if should_hoist?(node) + hoisted = trim_hoist_element node + hoisted.each { |child| trim_process_node child } + elsif should_delete?(node) + node.remove + else + if children = node.children + children.each { |child| trim_process_node child } + end + end + + node + end + + def trim_hoist_element(element) + hoisted = [] + element.children.each do |child| + element.before(child) + hoisted << child + end + element.remove + hoisted + end + + def should_hoist?(node) + return false unless node.element? + HTML_HOIST_ELEMENTS.include? node.name + end + + def should_delete?(node) + return true if HTML_DELETE_ELEMENT_TYPES.include? node.type + return true if node.element? && node.name == 'head' + return true if node.text? && node.text.strip.blank? + + false + end + end + end +end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb new file mode 100644 index 00000000000..c46fce6afe2 --- /dev/null +++ b/lib/gitlab/email/receiver.rb @@ -0,0 +1,101 @@ +# Inspired in great part by Discourse's Email::Receiver +module Gitlab + module Email + class Receiver + class ProcessingError < StandardError; end + class EmailUnparsableError < ProcessingError; end + class EmptyEmailError < ProcessingError; end + class UserNotFoundError < ProcessingError; end + class UserNotAuthorizedError < ProcessingError; end + class NoteableNotFoundError < ProcessingError; end + class AutoGeneratedEmailError < ProcessingError; end + class SentNotificationNotFound < ProcessingError; end + class InvalidNote < ProcessingError; end + + def initialize(raw) + @raw = raw + end + + def message + @message ||= Mail::Message.new(@raw) + rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e + raise EmailUnparsableError, e + end + + def execute + raise SentNotificationNotFound unless sent_notification + + raise EmptyEmailError if @raw.blank? + + raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/ + + author = sent_notification.recipient + + raise UserNotFoundError unless author + + project = sent_notification.project + + raise UserNotAuthorizedError unless author.can?(:create_note, project) + + raise NoteableNotFoundError unless sent_notification.noteable + + reply = ReplyParser.new(message).execute.strip + + raise EmptyEmailError if reply.blank? + + reply = add_attachments(reply) + + note = create_note(reply) + + unless note.persisted? + message = "The comment could not be created for the following reasons:" + note.errors.full_messages.each do |error| + message << "\n\n- #{error}" + end + + raise InvalidNote, message + end + end + + private + + def reply_key + reply_key = nil + message.to.each do |address| + reply_key = Gitlab::ReplyByEmail.reply_key_from_address(address) + break if reply_key + end + + reply_key + end + + def sent_notification + return nil unless reply_key + + SentNotification.for(reply_key) + end + + def add_attachments(reply) + attachments = AttachmentUploader.new(message).execute(project) + + attachments.each do |link| + text = "[#{link[:alt]}](#{link[:url]})" + text.prepend("!") if link[:is_image] + + reply << "\n\n#{text}" + end + end + + def create_note(reply) + Notes::CreateService.new( + sent_notification.project, + sent_notification.recipient, + note: reply, + noteable_type: sent_notification.noteable_type, + noteable_id: sent_notification.noteable_id, + commit_id: sent_notification.commit_id + ).execute + end + end + end +end diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb new file mode 100644 index 00000000000..6ceb755968c --- /dev/null +++ b/lib/gitlab/email/reply_parser.rb @@ -0,0 +1,91 @@ +# Inspired in great part by Discourse's Email::Receiver +module Gitlab + module Email + class ReplyParser + attr_accessor :message + + def initialize(message) + @message = message + end + + def execute + body = select_body(message) + + encoding = body.encoding + + body = discourse_email_trimmer(body) + + body = EmailReplyParser.parse_reply(body) + + body.force_encoding(encoding).encode("UTF-8") + end + + private + + def select_body(message) + html = nil + text = nil + + if message.multipart? + html = fix_charset(message.html_part) + text = fix_charset(message.text_part) + elsif message.content_type =~ /text\/html/ + html = fix_charset(message) + end + + # prefer plain text + return text if text + + if html + body = HtmlCleaner.new(html).output_html + else + body = fix_charset(message) + end + + # Certain trigger phrases that means we didn't parse correctly + if body =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/ + return "" + end + + body + end + + # Force encoding to UTF-8 on a Mail::Message or Mail::Part + def fix_charset(object) + return nil if object.nil? + + if object.charset + object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s + else + object.body.to_s + end + rescue + nil + end + + REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date) + REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" }) + + def discourse_email_trimmer(body) + lines = body.scrub.lines.to_a + range_end = 0 + + lines.each_with_index do |l, idx| + # This one might be controversial but so many reply lines have years, times and end with a colon. + # Let's try it and see how well it works. + break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) || + (l =~ /On \w+ \d+,? \d+,?.*wrote:/) + + # Headers on subsequent lines + break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX } + # Headers on the same line + break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3 + + range_end = idx + end + + lines[0..range_end].join.strip + end + end + end +end diff --git a/lib/gitlab/email_html_cleaner.rb b/lib/gitlab/email_html_cleaner.rb deleted file mode 100644 index 6d7a17fe87c..00000000000 --- a/lib/gitlab/email_html_cleaner.rb +++ /dev/null @@ -1,133 +0,0 @@ -# Taken mostly from Discourse's Email::HtmlCleaner -module Gitlab - # HtmlCleaner cleans up the extremely dirty HTML that many email clients - # generate by stripping out any excess divs or spans, removing styling in - # the process (which also makes the html more suitable to be parsed as - # Markdown). - class EmailHtmlCleaner - # Elements to hoist all children out of - HTML_HOIST_ELEMENTS = %w(div span font table tbody th tr td) - # Node types to always delete - HTML_DELETE_ELEMENT_TYPES = [ - Nokogiri::XML::Node::DTD_NODE, - Nokogiri::XML::Node::COMMENT_NODE, - ] - - # Private variables: - # @doc - nokogiri document - # @out - same as @doc, but only if trimming has occured - def initialize(html) - if html.is_a?(String) - @doc = Nokogiri::HTML(html) - else - @doc = html - end - end - - class << self - # EmailHtmlCleaner.trim(inp, opts={}) - # - # Arguments: - # inp - Either a HTML string or a Nokogiri document. - # Options: - # :return => :doc, :string - # Specify the desired return type. - # Defaults to the type of the input. - # A value of :string is equivalent to calling get_document_text() - # on the returned document. - def trim(inp, opts={}) - cleaner = EmailHtmlCleaner.new(inp) - - opts[:return] ||= (inp.is_a?(String) ? :string : :doc) - - if opts[:return] == :string - cleaner.output_html - else - cleaner.output_document - end - end - - # EmailHtmlCleaner.get_document_text(doc) - # - # Get the body portion of the document, including html, as a string. - def get_document_text(doc) - body = doc.xpath('//body') - if body - body.inner_html - else - doc.inner_html - end - end - end - - def output_document - @out ||= begin - doc = @doc - trim_process_node doc - add_newlines doc - doc - end - end - - def output_html - EmailHtmlCleaner.get_document_text(output_document) - end - - private - - def add_newlines(doc) - # Replace <br> tags with a markdown \n - doc.xpath('//br').each do |br| - br.replace(new_linebreak_node doc, 2) - end - # Surround <p> tags with newlines, to help with line-wise postprocessing - # and ensure markdown paragraphs - doc.xpath('//p').each do |p| - p.before(new_linebreak_node doc) - p.after(new_linebreak_node doc, 2) - end - end - - def new_linebreak_node(doc, count=1) - Nokogiri::XML::Text.new("\n" * count, doc) - end - - def trim_process_node(node) - if should_hoist?(node) - hoisted = trim_hoist_element node - hoisted.each { |child| trim_process_node child } - elsif should_delete?(node) - node.remove - else - if children = node.children - children.each { |child| trim_process_node child } - end - end - - node - end - - def trim_hoist_element(element) - hoisted = [] - element.children.each do |child| - element.before(child) - hoisted << child - end - element.remove - hoisted - end - - def should_hoist?(node) - return false unless node.element? - HTML_HOIST_ELEMENTS.include? node.name - end - - def should_delete?(node) - return true if HTML_DELETE_ELEMENT_TYPES.include? node.type - return true if node.element? && node.name == 'head' - return true if node.text? && node.text.strip.blank? - - false - end - end -end diff --git a/lib/gitlab/email_receiver.rb b/lib/gitlab/email_receiver.rb deleted file mode 100644 index 3c1f346c0cf..00000000000 --- a/lib/gitlab/email_receiver.rb +++ /dev/null @@ -1,192 +0,0 @@ -# Inspired in great part by Discourse's Email::Receiver -module Gitlab - class EmailReceiver - class ProcessingError < StandardError; end - class EmailUnparsableError < ProcessingError; end - class EmptyEmailError < ProcessingError; end - class UserNotFoundError < ProcessingError; end - class UserNotAuthorizedError < ProcessingError; end - class NoteableNotFoundError < ProcessingError; end - class AutoGeneratedEmailError < ProcessingError; end - class SentNotificationNotFound < ProcessingError; end - class InvalidNote < ProcessingError; end - - def initialize(raw) - @raw = raw - end - - def message - @message ||= Mail::Message.new(@raw) - rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e - raise EmailUnparsableError, e - end - - def execute - raise SentNotificationNotFound unless sent_notification - - raise EmptyEmailError if @raw.blank? - - raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/ - - author = sent_notification.recipient - - raise UserNotFoundError unless author - - project = sent_notification.project - - raise UserNotAuthorizedError unless author.can?(:create_note, project) - - raise NoteableNotFoundError unless sent_notification.noteable - - body = parse_body(message) - - upload_attachments.each do |link| - body << "\n\n#{link}" - end - - note = Notes::CreateService.new( - project, - author, - note: body, - noteable_type: sent_notification.noteable_type, - noteable_id: sent_notification.noteable_id, - commit_id: sent_notification.commit_id - ).execute - - unless note.persisted? - message = "The comment could not be created for the following reasons:" - note.errors.full_messages.each do |error| - message << "\n\n- #{error}" - end - raise InvalidNote, message - end - end - - def parse_body(message) - body = select_body(message) - - encoding = body.encoding - raise EmptyEmailError if body.strip.blank? - - body = discourse_email_trimmer(body) - raise EmptyEmailError if body.strip.blank? - - body = EmailReplyParser.parse_reply(body) - raise EmptyEmailError if body.strip.blank? - - body.force_encoding(encoding).encode("UTF-8") - end - - private - - def reply_key - reply_key = nil - message.to.each do |address| - reply_key = Gitlab::ReplyByEmail.reply_key_from_address(address) - break if reply_key - end - - reply_key - end - - def sent_notification - return nil unless reply_key - - SentNotification.for(reply_key) - end - - def select_body(message) - html = nil - text = nil - - if message.multipart? - html = fix_charset(message.html_part) - text = fix_charset(message.text_part) - elsif message.content_type =~ /text\/html/ - html = fix_charset(message) - end - - # prefer plain text - return text if text - - if html - body = EmailHtmlCleaner.new(html).output_html - else - body = fix_charset(message) - end - - # Certain trigger phrases that means we didn't parse correctly - if body =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/ - raise EmptyEmailError - end - - body - end - - # Force encoding to UTF-8 on a Mail::Message or Mail::Part - def fix_charset(object) - return nil if object.nil? - - if object.charset - object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s - else - object.body.to_s - end - rescue - nil - end - - REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date) - REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" }) - - def discourse_email_trimmer(body) - lines = body.scrub.lines.to_a - range_end = 0 - - lines.each_with_index do |l, idx| - # This one might be controversial but so many reply lines have years, times and end with a colon. - # Let's try it and see how well it works. - break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) || - (l =~ /On \w+ \d+,? \d+,?.*wrote:/) - - # Headers on subsequent lines - break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX } - # Headers on the same line - break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3 - - range_end = idx - end - - lines[0..range_end].join.strip - end - - def upload_attachments - attachments = [] - - message.attachments.each do |attachment| - tmp = Tempfile.new("gitlab-email-attachment") - begin - File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded } - - file = { - tempfile: tmp, - filename: attachment.filename, - content_type: attachment.content_type - } - - link = ::Projects::UploadService.new(sent_notification.project, file).execute - if link - text = "[#{link[:alt]}](#{link[:url]})" - text.prepend("!") if link[:is_image] - - attachments << text - end - ensure - tmp.close! - end - end - - attachments - end - end -end diff --git a/spec/lib/gitlab/email/reply_parser_spec.rb b/spec/lib/gitlab/email/reply_parser_spec.rb new file mode 100644 index 00000000000..999515beb1c --- /dev/null +++ b/spec/lib/gitlab/email/reply_parser_spec.rb @@ -0,0 +1,191 @@ +require "spec_helper" + +# Inspired in great part by Discourse's Email::Receiver +describe Gitlab::Email::ReplyParser do + def fixture_file(filename) + return '' if filename.blank? + file_path = File.expand_path(Rails.root + 'spec/fixtures/' + filename) + File.read(file_path) + end + + describe 'self.parse_body' do + def test_parse_body(mail_string) + described_class.new(Mail::Message.new(mail_string)).execute + end + + it "returns an empty string if the message is blank" do + expect(test_parse_body("")).to eq("") + end + + it "returns an empty string if the message is not an email" do + expect(test_parse_body("asdf" * 30)).to eq("") + end + + it "returns an empty string if there is no reply content" do + expect(test_parse_body(fixture_file("emails/no_content_reply.eml"))).to eq("") + end + + it "can parse the html section" do + expect(test_parse_body(fixture_file("emails/html_only.eml"))).to eq("The EC2 instance - I've seen that there tends to be odd and " + + "unrecommended settings on the Bitnami installs that I've checked out.") + end + + it "supports a Dutch reply" do + expect(test_parse_body(fixture_file("emails/dutch.eml"))).to eq("Dit is een antwoord in het Nederlands.") + end + + it "removes an 'on date wrote' quoting line" do + expect(test_parse_body(fixture_file("emails/on_wrote.eml"))).to eq("Sure, all you need to do is frobnicate the foobar and you'll be all set!") + end + + it "handles multiple paragraphs" do + expect(test_parse_body(fixture_file("emails/paragraphs.eml"))). + to eq( +"Is there any reason the *old* candy can't be be kept in silos while the new candy +is imported into *new* silos? + +The thing about candy is it stays delicious for a long time -- we can just keep +it there without worrying about it too much, imo. + +Thanks for listening." + ) + end + + it "handles multiple paragraphs when parsing html" do + expect(test_parse_body(fixture_file("emails/html_paragraphs.eml"))). + to eq( +"Awesome! + +Pleasure to have you here! + +:boom:" + ) + end + + it "handles newlines" do + expect(test_parse_body(fixture_file("emails/newlines.eml"))). + to eq( +"This is my reply. +It is my best reply. +It will also be my *only* reply." + ) + end + + it "handles inline reply" do + expect(test_parse_body(fixture_file("emails/inline_reply.eml"))). + to eq( +"On Wed, Oct 8, 2014 at 11:12 AM, techAPJ <info@unconfigured.discourse.org> wrote: + +> techAPJ <https://meta.discourse.org/users/techapj> +> November 28 +> +> Test reply. +> +> First paragraph. +> +> Second paragraph. +> +> To respond, reply to this email or visit +> https://meta.discourse.org/t/testing-default-email-replies/22638/3 in +> your browser. +> ------------------------------ +> Previous Replies codinghorror +> <https://meta.discourse.org/users/codinghorror> +> November 28 +> +> We're testing the latest GitHub email processing library which we are +> integrating now. +> +> https://github.com/github/email_reply_parser +> +> Go ahead and reply to this topic and I'll reply from various email clients +> for testing. +> ------------------------------ +> +> To respond, reply to this email or visit +> https://meta.discourse.org/t/testing-default-email-replies/22638/3 in +> your browser. +> +> To unsubscribe from these emails, visit your user preferences +> <https://meta.discourse.org/my/preferences>. +> + +The quick brown fox jumps over the lazy dog. The quick brown fox jumps over +the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown +fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. +The quick brown fox jumps over the lazy dog. The quick brown fox jumps over +the lazy dog. The quick brown fox jumps over the lazy dog." + ) + end + + it "strips iPhone signature" do + expect(test_parse_body(fixture_file("emails/iphone_signature.eml"))).not_to match /Sent from my iPhone/ + end + + it "properly renders email reply from gmail web client" do + expect(test_parse_body(fixture_file("emails/gmail_web.eml"))). + to eq( +"### This is a reply from standard GMail in Google Chrome. + +The quick brown fox jumps over the lazy dog. The quick brown fox jumps over +the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown +fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. +The quick brown fox jumps over the lazy dog. The quick brown fox jumps over +the lazy dog. The quick brown fox jumps over the lazy dog. + +Here's some **bold** text in Markdown. + +Here's a link http://example.com" + ) + end + + it "properly renders email reply from iOS default mail client" do + expect(test_parse_body(fixture_file("emails/ios_default.eml"))). + to eq( +"### this is a reply from iOS default mail + +The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. + +Here's some **bold** markdown text. + +Here's a link http://example.com" + ) + end + + it "properly renders email reply from Android 5 gmail client" do + expect(test_parse_body(fixture_file("emails/android_gmail.eml"))). + to eq( +"### this is a reply from Android 5 gmail + +The quick brown fox jumps over the lazy dog. The quick brown fox jumps over +the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown +fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. +The quick brown fox jumps over the lazy dog. + +This is **bold** in Markdown. + +This is a link to http://example.com" + ) + end + + it "properly renders email reply from Windows 8.1 Metro default mail client" do + expect(test_parse_body(fixture_file("emails/windows_8_metro.eml"))). + to eq( +"### reply from default mail client in Windows 8.1 Metro + + +The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. + + +This is a **bold** word in Markdown + + +This is a link http://example.com" + ) + end + + it "properly renders email reply from MS Outlook client" do + expect(test_parse_body(fixture_file("emails/outlook.eml"))).to eq("Microsoft Outlook 2010") + end + end +end diff --git a/spec/lib/gitlab/email_receiver_spec.rb b/spec/lib/gitlab/email_receiver_spec.rb deleted file mode 100644 index d6302eac13a..00000000000 --- a/spec/lib/gitlab/email_receiver_spec.rb +++ /dev/null @@ -1,481 +0,0 @@ -require "spec_helper" - -# Inspired in great part by Discourse's Email::Receiver -describe Gitlab::EmailReceiver do - def fixture_file(filename) - return '' if filename.blank? - file_path = File.expand_path(Rails.root + 'spec/fixtures/' + filename) - File.read(file_path) - end - - before do - allow(Gitlab.config.reply_by_email).to receive(:enabled).and_return(true) - allow(Gitlab.config.reply_by_email).to receive(:address).and_return("reply+%{reply_key}@appmail.adventuretime.ooo") - end - - describe 'parse_body' do - def test_parse_body(mail_string) - Gitlab::EmailReceiver.new(nil).parse_body(Mail::Message.new(mail_string)) - end - - it "raises EmptyEmailError if the message is blank" do - expect { test_parse_body("") }.to raise_error(Gitlab::EmailReceiver::EmptyEmailError) - end - - it "raises EmptyEmailError if the message is not an email" do - expect { test_parse_body("asdf" * 30) }.to raise_error(Gitlab::EmailReceiver::EmptyEmailError) - end - - it "raises EmptyEmailError if there is no reply content" do - expect { test_parse_body(fixture_file("emails/no_content_reply.eml")) }.to raise_error(Gitlab::EmailReceiver::EmptyEmailError) - end - - it "can parse the html section" do - expect(test_parse_body(fixture_file("emails/html_only.eml"))).to eq("The EC2 instance - I've seen that there tends to be odd and " + - "unrecommended settings on the Bitnami installs that I've checked out.") - end - - it "supports a Dutch reply" do - expect(test_parse_body(fixture_file("emails/dutch.eml"))).to eq("Dit is een antwoord in het Nederlands.") - end - - it "removes an 'on date wrote' quoting line" do - expect(test_parse_body(fixture_file("emails/on_wrote.eml"))).to eq("Sure, all you need to do is frobnicate the foobar and you'll be all set!") - end - - it "handles multiple paragraphs" do - expect(test_parse_body(fixture_file("emails/paragraphs.eml"))). - to eq( -"Is there any reason the *old* candy can't be be kept in silos while the new candy -is imported into *new* silos? - -The thing about candy is it stays delicious for a long time -- we can just keep -it there without worrying about it too much, imo. - -Thanks for listening." - ) - end - - it "handles multiple paragraphs when parsing html" do - expect(test_parse_body(fixture_file("emails/html_paragraphs.eml"))). - to eq( -"Awesome! - -Pleasure to have you here! - -:boom:" - ) - end - - it "handles newlines" do - expect(test_parse_body(fixture_file("emails/newlines.eml"))). - to eq( -"This is my reply. -It is my best reply. -It will also be my *only* reply." - ) - end - - it "handles inline reply" do - expect(test_parse_body(fixture_file("emails/inline_reply.eml"))). - to eq( -"On Wed, Oct 8, 2014 at 11:12 AM, techAPJ <info@unconfigured.discourse.org> wrote: - -> techAPJ <https://meta.discourse.org/users/techapj> -> November 28 -> -> Test reply. -> -> First paragraph. -> -> Second paragraph. -> -> To respond, reply to this email or visit -> https://meta.discourse.org/t/testing-default-email-replies/22638/3 in -> your browser. -> ------------------------------ -> Previous Replies codinghorror -> <https://meta.discourse.org/users/codinghorror> -> November 28 -> -> We're testing the latest GitHub email processing library which we are -> integrating now. -> -> https://github.com/github/email_reply_parser -> -> Go ahead and reply to this topic and I'll reply from various email clients -> for testing. -> ------------------------------ -> -> To respond, reply to this email or visit -> https://meta.discourse.org/t/testing-default-email-replies/22638/3 in -> your browser. -> -> To unsubscribe from these emails, visit your user preferences -> <https://meta.discourse.org/my/preferences>. -> - -The quick brown fox jumps over the lazy dog. The quick brown fox jumps over -the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown -fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. -The quick brown fox jumps over the lazy dog. The quick brown fox jumps over -the lazy dog. The quick brown fox jumps over the lazy dog." - ) - end - - it "strips iPhone signature" do - expect(test_parse_body(fixture_file("emails/iphone_signature.eml"))).not_to match /Sent from my iPhone/ - end - - it "properly renders email reply from gmail web client" do - expect(test_parse_body(fixture_file("emails/gmail_web.eml"))). - to eq( -"### This is a reply from standard GMail in Google Chrome. - -The quick brown fox jumps over the lazy dog. The quick brown fox jumps over -the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown -fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. -The quick brown fox jumps over the lazy dog. The quick brown fox jumps over -the lazy dog. The quick brown fox jumps over the lazy dog. - -Here's some **bold** text in Markdown. - -Here's a link http://example.com" - ) - end - - it "properly renders email reply from iOS default mail client" do - expect(test_parse_body(fixture_file("emails/ios_default.eml"))). - to eq( -"### this is a reply from iOS default mail - -The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. - -Here's some **bold** markdown text. - -Here's a link http://example.com" - ) - end - - it "properly renders email reply from Android 5 gmail client" do - expect(test_parse_body(fixture_file("emails/android_gmail.eml"))). - to eq( -"### this is a reply from Android 5 gmail - -The quick brown fox jumps over the lazy dog. The quick brown fox jumps over -the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown -fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. -The quick brown fox jumps over the lazy dog. - -This is **bold** in Markdown. - -This is a link to http://example.com" - ) - end - - it "properly renders email reply from Windows 8.1 Metro default mail client" do - expect(test_parse_body(fixture_file("emails/windows_8_metro.eml"))). - to eq( -"### reply from default mail client in Windows 8.1 Metro - - -The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. The quick brown fox jumps over the lazy dog. - - -This is a **bold** word in Markdown - - -This is a link http://example.com" - ) - end - - it "properly renders email reply from MS Outlook client" do - expect(test_parse_body(fixture_file("emails/outlook.eml"))).to eq("Microsoft Outlook 2010") - end - end - -# describe "posting replies" do -# let(:reply_key) { raise "Override this in a lower describe block" } -# let(:email_raw) { raise "Override this in a lower describe block" } -# # ---- -# let(:receiver) { Gitlab::EmailReceiver.new(email_raw) } -# let(:post) { create_post } -# let(:topic) { post.topic } -# let(:posting_user) { post.user } -# let(:replying_user_email) { 'jake@adventuretime.ooo' } -# let(:replying_user) { Fabricate(:user, email: replying_user_email, trust_level: 2)} -# let(:email_log) { EmailLog.new(reply_key: reply_key, -# post: post, -# post_id: post.id, -# topic_id: post.topic_id, -# email_type: 'user_posted', -# user: replying_user, -# user_id: replying_user.id, -# to_address: replying_user_email -# ) } - -# before do -# email_log.save -# end - -# # === Success Posting === - -# describe "valid_reply.eml" do -# let!(:reply_key) { '59d8df8370b7e95c5a49fbf86aeb2c93' } -# let!(:email_raw) { fixture_file("emails/valid_reply.eml") } - -# it "creates a post with the correct content" do -# start_count = topic.posts.count - -# receiver.process - -# expect(topic.posts.count).to eq(start_count + 1) -# created_post = topic.posts.last -# expect(created_post.via_email).to eq(true) -# expect(created_post.raw_email).to eq(fixture_file("emails/valid_reply.eml")) -# expect(created_post.cooked.strip).to eq(fixture_file("emails/valid_reply.cooked").strip) -# end -# end - -# describe "paragraphs.eml" do -# let!(:reply_key) { '59d8df8370b7e95c5a49fbf86aeb2c93' } -# let!(:email_raw) { fixture_file("emails/paragraphs.eml") } - -# it "cooks multiple paragraphs with traditional Markdown linebreaks" do -# start_count = topic.posts.count - -# receiver.process - -# expect(topic.posts.count).to eq(start_count + 1) -# expect(topic.posts.last.cooked.strip).to eq(fixture_file("emails/paragraphs.cooked").strip) -# expect(topic.posts.last.cooked).not_to match /<br/ -# end -# end - -# describe "attachment.eml" do -# let!(:reply_key) { '636ca428858779856c226bb145ef4fad' } -# let!(:email_raw) { -# fixture_file("emails/attachment.eml") -# .gsub("TO", "reply+#{reply_key}@appmail.adventuretime.ooo") -# .gsub("FROM", replying_user_email) -# } - -# let(:upload_sha) { '04df605be528d03876685c52166d4b063aabb78a' } - -# it "creates a post with an attachment" do -# Upload.stubs(:fix_image_orientation) -# ImageOptim.any_instance.stubs(:optimize_image!) - -# start_count = topic.posts.count -# Upload.find_by(sha1: upload_sha).try(:destroy) - -# receiver.process - -# expect(topic.posts.count).to eq(start_count + 1) -# expect(topic.posts.last.cooked).to match /<img src=['"](\/uploads\/default\/original\/.+\.png)['"] width=['"]289['"] height=['"]126['"]>/ -# expect(Upload.find_by(sha1: upload_sha)).not_to eq(nil) -# end - -# end - -# # === Failure Conditions === - -# describe "too_short.eml" do -# let!(:reply_key) { '636ca428858779856c226bb145ef4fad' } -# let!(:email_raw) { -# fixture_file("emails/too_short.eml") -# .gsub("TO", "reply+#{reply_key}@appmail.adventuretime.ooo") -# .gsub("FROM", replying_user_email) -# .gsub("SUBJECT", "re: [Discourse Meta] eviltrout posted in 'Adventure Time Sux'") -# } - -# it "raises an InvalidPost error" do -# SiteSetting.min_post_length = 5 -# expect { receiver.process }.to raise_error(Gitlab::EmailReceiver::InvalidPost) -# end -# end - -# describe "too_many_mentions.eml" do -# let!(:reply_key) { '636ca428858779856c226bb145ef4fad' } -# let!(:email_raw) { fixture_file("emails/too_many_mentions.eml") } - -# it "raises an InvalidPost error" do -# SiteSetting.max_mentions_per_post = 10 -# (1..11).each do |i| -# Fabricate(:user, username: "user#{i}").save -# end - -# expect { receiver.process }.to raise_error(Gitlab::EmailReceiver::InvalidPost) -# end -# end - -# describe "auto response email replies should not be accepted" do -# let!(:reply_key) { '636ca428858779856c226bb145ef4fad' } -# let!(:email_raw) { fixture_file("emails/auto_reply.eml") } -# it "raises a AutoGeneratedEmailError" do -# expect { receiver.process }.to raise_error(Gitlab::EmailReceiver::AutoGeneratedEmailError) -# end -# end - -# end - -# describe "posting reply to a closed topic" do -# let(:reply_key) { raise "Override this in a lower describe block" } -# let(:email_raw) { raise "Override this in a lower describe block" } -# let(:receiver) { Gitlab::EmailReceiver.new(email_raw) } -# let(:topic) { Fabricate(:topic, closed: true) } -# let(:post) { Fabricate(:post, topic: topic, post_number: 1) } -# let(:replying_user_email) { 'jake@adventuretime.ooo' } -# let(:replying_user) { Fabricate(:user, email: replying_user_email, trust_level: 2) } -# let(:email_log) { EmailLog.new(reply_key: reply_key, -# post: post, -# post_id: post.id, -# topic_id: topic.id, -# email_type: 'user_posted', -# user: replying_user, -# user_id: replying_user.id, -# to_address: replying_user_email -# ) } - -# before do -# email_log.save -# end - -# describe "should not create post" do -# let!(:reply_key) { '59d8df8370b7e95c5a49fbf86aeb2c93' } -# let!(:email_raw) { fixture_file("emails/valid_reply.eml") } -# it "raises a TopicClosedError" do -# expect { receiver.process }.to raise_error(Gitlab::EmailReceiver::TopicClosedError) -# end -# end -# end - -# describe "posting reply to a deleted topic" do -# let(:reply_key) { raise "Override this in a lower describe block" } -# let(:email_raw) { raise "Override this in a lower describe block" } -# let(:receiver) { Gitlab::EmailReceiver.new(email_raw) } -# let(:deleted_topic) { Fabricate(:deleted_topic) } -# let(:post) { Fabricate(:post, topic: deleted_topic, post_number: 1) } -# let(:replying_user_email) { 'jake@adventuretime.ooo' } -# let(:replying_user) { Fabricate(:user, email: replying_user_email, trust_level: 2) } -# let(:email_log) { EmailLog.new(reply_key: reply_key, -# post: post, -# post_id: post.id, -# topic_id: deleted_topic.id, -# email_type: 'user_posted', -# user: replying_user, -# user_id: replying_user.id, -# to_address: replying_user_email -# ) } - -# before do -# email_log.save -# end - -# describe "should not create post" do -# let!(:reply_key) { '59d8df8370b7e95c5a49fbf86aeb2c93' } -# let!(:email_raw) { fixture_file("emails/valid_reply.eml") } -# it "raises a TopicNotFoundError" do -# expect { receiver.process }.to raise_error(Gitlab::EmailReceiver::TopicNotFoundError) -# end -# end -# end - -# describe "posting a new topic" do -# let(:category_destination) { raise "Override this in a lower describe block" } -# let(:email_raw) { raise "Override this in a lower describe block" } -# let(:allow_strangers) { false } -# # ---- -# let(:receiver) { Gitlab::EmailReceiver.new(email_raw) } -# let(:user_email) { 'jake@adventuretime.ooo' } -# let(:user) { Fabricate(:user, email: user_email, trust_level: 2)} -# let(:category) { Fabricate(:category, email_in: category_destination, email_in_allow_strangers: allow_strangers) } - -# before do -# SiteSetting.email_in = true -# user.save -# category.save -# end - -# describe "too_short.eml" do -# let!(:category_destination) { 'incoming+amazing@appmail.adventuretime.ooo' } -# let(:email_raw) { -# fixture_file("emails/too_short.eml") -# .gsub("TO", category_destination) -# .gsub("FROM", user_email) -# .gsub("SUBJECT", "A long subject that passes the checks") -# } - -# it "does not create a topic if the post fails" do -# before_topic_count = Topic.count - -# expect { receiver.process }.to raise_error(Gitlab::EmailReceiver::InvalidPost) - -# expect(Topic.count).to eq(before_topic_count) -# end - -# end - -# end - -# def fill_email(mail, from, to, body = nil, subject = nil) -# result = mail.gsub("FROM", from).gsub("TO", to) -# if body -# result.gsub!(/Hey.*/m, body) -# end -# if subject -# result.sub!(/We .*/, subject) -# end -# result -# end - -# def process_email(opts) -# incoming_email = fixture_file("emails/valid_incoming.eml") -# email = fill_email(incoming_email, opts[:from], opts[:to], opts[:body], opts[:subject]) -# Gitlab::EmailReceiver.new(email).process -# end - -# describe "with a valid email" do -# let(:reply_key) { "59d8df8370b7e95c5a49fbf86aeb2c93" } -# let(:to) { SiteSetting.reply_by_email_address.gsub("%{reply_key}", reply_key) } - -# let(:valid_reply) { -# reply = fixture_file("emails/valid_reply.eml") -# to = SiteSetting.reply_by_email_address.gsub("%{reply_key}", reply_key) -# fill_email(reply, "test@test.com", to) -# } - -# let(:receiver) { Gitlab::EmailReceiver.new(valid_reply) } -# let(:post) { create_post } -# let(:user) { post.user } -# let(:email_log) { EmailLog.new(reply_key: reply_key, -# post_id: post.id, -# topic_id: post.topic_id, -# user_id: post.user_id, -# post: post, -# user: user, -# email_type: 'test', -# to_address: 'test@test.com' -# ) } -# let(:reply_body) { -# "I could not disagree more. I am obviously biased but adventure time is the -# greatest show ever created. Everyone should watch it. - -# - Jake out" } - -# describe "with an email log" do - -# it "extracts data" do -# expect{ receiver.process }.to raise_error(Gitlab::EmailReceiver::EmailLogNotFound) - -# email_log.save! -# receiver.process - -# expect(receiver.body).to eq(reply_body) -# expect(receiver.email_log).to eq(email_log) -# end - -# end - -# end -end |