From f76eac56b9d7d4ae61010cddcca68682824b2239 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Tue, 18 Aug 2015 15:46:36 -0700
Subject: Reply by email POC

---
 lib/gitlab/email_receiver.rb | 50 ++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)
 create mode 100644 lib/gitlab/email_receiver.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/email_receiver.rb b/lib/gitlab/email_receiver.rb
new file mode 100644
index 00000000000..a9f67bb5da0
--- /dev/null
+++ b/lib/gitlab/email_receiver.rb
@@ -0,0 +1,50 @@
+module Gitlab
+  class EmailReceiver
+    def initialize(raw)
+      @raw = raw
+    end
+
+    def message
+      @message ||= Mail::Message.new(@raw)
+    end
+
+    def process
+      return unless message && sent_notification
+
+      Notes::CreateService.new(
+        sent_notification.project,
+        sent_notification.recipient,
+        note:           message.text_part.to_s,
+        noteable_type:  sent_notification.noteable_type,
+        noteable_id:    sent_notification.noteable_id,
+        commit_id:      sent_notification.commit_id
+      ).execute
+    end
+
+    private
+
+    def reply_key
+      address = Gitlab.config.reply_by_email.address
+      return nil unless address
+
+      regex = Regexp.escape(address)
+      regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.*)")
+      regex = Regexp.new(regex)
+
+      address = message.to.find { |address| address =~ regex }
+      return nil unless address
+
+      match = address.match(regex)
+        
+      return nil unless match && match[1].present?
+
+      match[1]
+    end
+
+    def sent_notification
+      return nil unless reply_key
+      
+      SentNotification.for(reply_key)
+    end
+  end
+end
-- 
cgit v1.2.3


From 8906cabae7a6be44cafcedcaf27352614fcc462b Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Tue, 18 Aug 2015 17:02:26 -0700
Subject: Changes and stuff.

---
 lib/gitlab/email_html_cleaner.rb | 133 ++++++++++++++++++++++++++++++++++++
 lib/gitlab/email_receiver.rb     | 144 +++++++++++++++++++++++++++++++++------
 lib/gitlab/reply_by_email.rb     |  47 +++++++++++++
 3 files changed, 305 insertions(+), 19 deletions(-)
 create mode 100644 lib/gitlab/email_html_cleaner.rb
 create mode 100644 lib/gitlab/reply_by_email.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/email_html_cleaner.rb b/lib/gitlab/email_html_cleaner.rb
new file mode 100644
index 00000000000..6d7a17fe87c
--- /dev/null
+++ b/lib/gitlab/email_html_cleaner.rb
@@ -0,0 +1,133 @@
+# Taken mostly from Discourse's Email::HtmlCleaner
+module Gitlab
+  # HtmlCleaner cleans up the extremely dirty HTML that many email clients
+  # generate by stripping out any excess divs or spans, removing styling in
+  # the process (which also makes the html more suitable to be parsed as
+  # Markdown).
+  class EmailHtmlCleaner
+    # Elements to hoist all children out of
+    HTML_HOIST_ELEMENTS = %w(div span font table tbody th tr td)
+    # Node types to always delete
+    HTML_DELETE_ELEMENT_TYPES = [
+      Nokogiri::XML::Node::DTD_NODE,
+      Nokogiri::XML::Node::COMMENT_NODE,
+    ]
+
+    # Private variables:
+    #   @doc - nokogiri document
+    #   @out - same as @doc, but only if trimming has occured
+    def initialize(html)
+      if html.is_a?(String)
+        @doc = Nokogiri::HTML(html)
+      else
+        @doc = html
+      end
+    end
+
+    class << self
+      # EmailHtmlCleaner.trim(inp, opts={})
+      #
+      # Arguments:
+      #   inp - Either a HTML string or a Nokogiri document.
+      # Options:
+      #   :return => :doc, :string
+      #     Specify the desired return type.
+      #     Defaults to the type of the input.
+      #     A value of :string is equivalent to calling get_document_text()
+      #     on the returned document.
+      def trim(inp, opts={})
+        cleaner = EmailHtmlCleaner.new(inp)
+
+        opts[:return] ||= (inp.is_a?(String) ? :string : :doc)
+
+        if opts[:return] == :string
+          cleaner.output_html
+        else
+          cleaner.output_document
+        end
+      end
+
+      # EmailHtmlCleaner.get_document_text(doc)
+      #
+      # Get the body portion of the document, including html, as a string.
+      def get_document_text(doc)
+        body = doc.xpath('//body')
+        if body
+          body.inner_html
+        else
+          doc.inner_html
+        end
+      end
+    end
+
+    def output_document
+      @out ||= begin
+        doc = @doc
+        trim_process_node doc
+        add_newlines doc
+        doc
+      end
+    end
+
+    def output_html
+      EmailHtmlCleaner.get_document_text(output_document)
+    end
+
+    private
+
+    def add_newlines(doc)
+      # Replace <br> tags with a markdown \n
+      doc.xpath('//br').each do |br|
+        br.replace(new_linebreak_node doc, 2)
+      end
+      # Surround <p> tags with newlines, to help with line-wise postprocessing
+      # and ensure markdown paragraphs
+      doc.xpath('//p').each do |p|
+        p.before(new_linebreak_node doc)
+        p.after(new_linebreak_node doc, 2)
+      end
+    end
+
+    def new_linebreak_node(doc, count=1)
+      Nokogiri::XML::Text.new("\n" * count, doc)
+    end
+
+    def trim_process_node(node)
+      if should_hoist?(node)
+        hoisted = trim_hoist_element node
+        hoisted.each { |child| trim_process_node child }
+      elsif should_delete?(node)
+        node.remove
+      else
+        if children = node.children
+          children.each { |child| trim_process_node child }
+        end
+      end
+
+      node
+    end
+
+    def trim_hoist_element(element)
+      hoisted = []
+      element.children.each do |child|
+        element.before(child)
+        hoisted << child
+      end
+      element.remove
+      hoisted
+    end
+
+    def should_hoist?(node)
+      return false unless node.element?
+      HTML_HOIST_ELEMENTS.include? node.name
+    end
+
+    def should_delete?(node)
+      return true if HTML_DELETE_ELEMENT_TYPES.include? node.type
+      return true if node.element? && node.name == 'head'
+      return true if node.text? && node.text.strip.blank?
+
+      false
+    end
+  end
+end
diff --git a/lib/gitlab/email_receiver.rb b/lib/gitlab/email_receiver.rb
index a9f67bb5da0..18a06d2ee8c 100644
--- a/lib/gitlab/email_receiver.rb
+++ b/lib/gitlab/email_receiver.rb
@@ -1,44 +1,69 @@
+# Inspired in great part by Discourse's Email::Receiver
 module Gitlab
   class EmailReceiver
+    class ProcessingError < StandardError; end
+    class EmailUnparsableError < ProcessingError; end
+    class EmptyEmailError < ProcessingError; end
+    class UserNotFoundError < ProcessingError; end
+    class UserNotAuthorizedLevelError < ProcessingError; end
+    class NoteableNotFoundError < ProcessingError; end
+    class AutoGeneratedEmailError < ProcessingError; end
+    class SentNotificationNotFound < ProcessingError; end
+    class InvalidNote < ProcessingError; end
+
     def initialize(raw)
       @raw = raw
     end
 
     def message
       @message ||= Mail::Message.new(@raw)
+    rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e
+      raise EmailUnparsableError, e
     end
 
     def process
-      return unless message && sent_notification
+      raise EmptyEmailError if @raw.blank?
+
+      raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/
+
+      raise SentNotificationNotFound unless sent_notification
+
+      author = sent_notification.recipient
+
+      raise UserNotFoundError unless author
+
+      project = sent_notification.project
+
+      raise UserNotAuthorizedLevelError unless author.can?(:create_note, project)
+
+      raise NoteableNotFoundError unless sent_notification.noteable
+
+      body = parse_body(message)
 
-      Notes::CreateService.new(
-        sent_notification.project,
-        sent_notification.recipient,
-        note:           message.text_part.to_s,
+      note = Notes::CreateService.new(
+        project,
+        author,
+        note:           body,
         noteable_type:  sent_notification.noteable_type,
         noteable_id:    sent_notification.noteable_id,
         commit_id:      sent_notification.commit_id
       ).execute
+
+      unless note.persisted?
+        raise InvalidNote, note.errors.full_messages.join("\n")
+      end
     end
 
     private
 
     def reply_key
-      address = Gitlab.config.reply_by_email.address
-      return nil unless address
-
-      regex = Regexp.escape(address)
-      regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.*)")
-      regex = Regexp.new(regex)
-
-      address = message.to.find { |address| address =~ regex }
-      return nil unless address
+      reply_key = nil
+      message.to.each do |address|
+        reply_key = Gitlab::ReplyByEmail.reply_key_from_address(address)
+        break if reply_key
+      end
 
-      match = address.match(regex)
-        
-      return nil unless match && match[1].present?
-
-      match[1]
+      reply_key
     end
 
     def sent_notification
@@ -46,5 +71,86 @@ module Gitlab
       
       SentNotification.for(reply_key)
     end
+
+    def parse_body(message)
+      body = select_body(message)
+
+      encoding = body.encoding
+      raise EmptyEmailError if body.strip.blank?
+
+      body = discourse_email_trimmer(body)
+      raise EmptyEmailError if body.strip.blank?
+
+      body = EmailReplyParser.parse_reply(body)
+      raise EmptyEmailError if body.strip.blank?
+
+      body.force_encoding(encoding).encode("UTF-8")
+    end
+
+    def select_body(message)
+      html = nil
+      text = nil
+
+      if message.multipart?
+        html = fix_charset(message.html_part)
+        text = fix_charset(message.text_part)
+      elsif message.content_type =~ /text\/html/
+        html = fix_charset(message)
+      end
+
+      # prefer plain text
+      return text if text
+
+      if html
+        body = EmailHtmlCleaner.new(html).output_html
+      else
+        body = fix_charset(message)
+      end
+
+      # Certain trigger phrases that means we didn't parse correctly
+      if body =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/
+        raise EmptyEmailError
+      end
+
+      body
+    end
+
+    # Force encoding to UTF-8 on a Mail::Message or Mail::Part
+    def fix_charset(object)
+      return nil if object.nil?
+
+      if object.charset
+        object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s
+      else
+        object.body.to_s
+      end
+    rescue
+      nil
+    end
+
+    REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date)
+    REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" })
+
+    def discourse_email_trimmer(body)
+      lines = body.scrub.lines.to_a
+      range_end = 0
+
+      lines.each_with_index do |l, idx|
+        break if l =~ /\A\s*\-{3,80}\s*\z/ ||
+                 # This one might be controversial but so many reply lines have years, times and end with a colon.
+                 # Let's try it and see how well it works.
+                 (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
+                 (l =~ /On \w+ \d+,? \d+,?.*wrote:/)
+
+        # Headers on subsequent lines
+        break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX }
+        # Headers on the same line
+        break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3
+
+        range_end = idx
+      end
+
+      lines[0..range_end].join.strip
+    end
   end
 end
diff --git a/lib/gitlab/reply_by_email.rb b/lib/gitlab/reply_by_email.rb
new file mode 100644
index 00000000000..b6157de3610
--- /dev/null
+++ b/lib/gitlab/reply_by_email.rb
@@ -0,0 +1,47 @@
+module Gitlab
+  module ReplyByEmail
+    class << self
+      def enabled?
+        config.enabled &&
+          config.address &&
+          config.address.include?("%{reply_key}")
+      end
+
+      def reply_key
+        return nil unless enabled?
+
+        SecureRandom.hex(16)
+      end
+
+      def reply_address(reply_key)
+        config.address.gsub('%{reply_key}', reply_key)
+      end
+
+      def reply_key_from_address(address)
+        return unless address_regex
+
+        match = address.match(address_regex)
+        return unless match
+
+        match[1]
+      end
+
+      private
+
+      def config
+        Gitlab.config.reply_by_email
+      end
+
+      def address_regex
+        @address_regex ||= begin
+          wildcard_address = config.address
+          return nil unless wildcard_address
+
+          regex = Regexp.escape(wildcard_address)
+          regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.+)")
+          Regexp.new(regex).freeze
+        end
+      end
+    end
+  end
+end
-- 
cgit v1.2.3


From ee1eb2948d30365be241bed6a68b29da57495af8 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Wed, 19 Aug 2015 10:18:05 -0700
Subject: Turn reply-by-email attachments into uploads.

---
 lib/gitlab/email_receiver.rb | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/email_receiver.rb b/lib/gitlab/email_receiver.rb
index 18a06d2ee8c..3dd8942a262 100644
--- a/lib/gitlab/email_receiver.rb
+++ b/lib/gitlab/email_receiver.rb
@@ -40,6 +40,10 @@ module Gitlab
 
       body = parse_body(message)
 
+      upload_attachments.each do |link|
+        body << "\n\n#{link}"
+      end
+
       note = Notes::CreateService.new(
         project,
         author,
@@ -152,5 +156,34 @@ module Gitlab
 
       lines[0..range_end].join.strip
     end
+
+    def upload_attachments
+      attachments = []
+
+      message.attachments.each do |attachment|
+        tmp = Tempfile.new("gitlab-email-attachment")
+        begin
+          File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded }
+
+          file = {
+            tempfile:     tmp,
+            filename:     attachment.filename,
+            content_type: attachment.content_type
+          }
+
+          link = ::Projects::UploadService.new(sent_notification.project, file).execute
+          if link
+            text = "[#{link[:alt]}](#{link[:url]})"
+            text.prepend("!") if link[:is_image]
+
+            attachments << text
+          end
+        ensure
+          tmp.close!
+        end
+      end
+
+      attachments
+    end
   end
 end
-- 
cgit v1.2.3


From 76dbafba86dda96b7ba2f93fc7e07eea3ca48302 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Wed, 19 Aug 2015 11:10:21 -0700
Subject: Send a rejection email when the incoming email couldn't be processed.

---
 lib/gitlab/email_receiver.rb | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/email_receiver.rb b/lib/gitlab/email_receiver.rb
index 3dd8942a262..a0b4ff87e02 100644
--- a/lib/gitlab/email_receiver.rb
+++ b/lib/gitlab/email_receiver.rb
@@ -5,7 +5,7 @@ module Gitlab
     class EmailUnparsableError < ProcessingError; end
     class EmptyEmailError < ProcessingError; end
     class UserNotFoundError < ProcessingError; end
-    class UserNotAuthorizedLevelError < ProcessingError; end
+    class UserNotAuthorizedError < ProcessingError; end
     class NoteableNotFoundError < ProcessingError; end
     class AutoGeneratedEmailError < ProcessingError; end
     class SentNotificationNotFound < ProcessingError; end
@@ -21,20 +21,20 @@ module Gitlab
       raise EmailUnparsableError, e
     end
 
-    def process
+    def execute
+      raise SentNotificationNotFound unless sent_notification
+
       raise EmptyEmailError if @raw.blank?
 
       raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/
 
-      raise SentNotificationNotFound unless sent_notification
-
       author = sent_notification.recipient
 
       raise UserNotFoundError unless author
 
       project = sent_notification.project
 
-      raise UserNotAuthorizedLevelError unless author.can?(:create_note, project)
+      raise UserNotAuthorizedError unless author.can?(:create_note, project)
 
       raise NoteableNotFoundError unless sent_notification.noteable
 
@@ -54,7 +54,11 @@ module Gitlab
       ).execute
 
       unless note.persisted?
-        raise InvalidNote, note.errors.full_messages.join("\n")
+        message = "The comment could not be created for the following reasons:"
+        note.errors.full_messages.each do |error|
+          message << "\n\n- #{error}"
+        end
+        raise InvalidNote, message
       end
     end
 
-- 
cgit v1.2.3


From 83081f167397c6c325a4f8c604191e766b2c3d3b Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Wed, 19 Aug 2015 18:00:13 -0700
Subject: Start on tests.

---
 lib/gitlab/email_receiver.rb | 37 ++++++++++++++++++-------------------
 1 file changed, 18 insertions(+), 19 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/email_receiver.rb b/lib/gitlab/email_receiver.rb
index a0b4ff87e02..3c1f346c0cf 100644
--- a/lib/gitlab/email_receiver.rb
+++ b/lib/gitlab/email_receiver.rb
@@ -62,6 +62,21 @@ module Gitlab
       end
     end
 
+    def parse_body(message)
+      body = select_body(message)
+
+      encoding = body.encoding
+      raise EmptyEmailError if body.strip.blank?
+
+      body = discourse_email_trimmer(body)
+      raise EmptyEmailError if body.strip.blank?
+
+      body = EmailReplyParser.parse_reply(body)
+      raise EmptyEmailError if body.strip.blank?
+
+      body.force_encoding(encoding).encode("UTF-8")
+    end
+
     private
 
     def reply_key
@@ -80,21 +95,6 @@ module Gitlab
       SentNotification.for(reply_key)
     end
 
-    def parse_body(message)
-      body = select_body(message)
-
-      encoding = body.encoding
-      raise EmptyEmailError if body.strip.blank?
-
-      body = discourse_email_trimmer(body)
-      raise EmptyEmailError if body.strip.blank?
-
-      body = EmailReplyParser.parse_reply(body)
-      raise EmptyEmailError if body.strip.blank?
-
-      body.force_encoding(encoding).encode("UTF-8")
-    end
-
     def select_body(message)
       html = nil
       text = nil
@@ -144,10 +144,9 @@ module Gitlab
       range_end = 0
 
       lines.each_with_index do |l, idx|
-        break if l =~ /\A\s*\-{3,80}\s*\z/ ||
-                 # This one might be controversial but so many reply lines have years, times and end with a colon.
-                 # Let's try it and see how well it works.
-                 (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
+        # This one might be controversial but so many reply lines have years, times and end with a colon.
+       # Let's try it and see how well it works.
+       break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
                  (l =~ /On \w+ \d+,? \d+,?.*wrote:/)
 
         # Headers on subsequent lines
-- 
cgit v1.2.3


From e9972efc2f3d730e989907585dd1438c517a0bba Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Thu, 20 Aug 2015 11:05:06 -0700
Subject: Extract ReplyParser and AttachmentUploader from Receiver.

---
 lib/gitlab/email/attachment_uploader.rb |  35 ++++++
 lib/gitlab/email/html_cleaner.rb        | 135 ++++++++++++++++++++++
 lib/gitlab/email/receiver.rb            | 101 +++++++++++++++++
 lib/gitlab/email/reply_parser.rb        |  91 +++++++++++++++
 lib/gitlab/email_html_cleaner.rb        | 133 ----------------------
 lib/gitlab/email_receiver.rb            | 192 --------------------------------
 6 files changed, 362 insertions(+), 325 deletions(-)
 create mode 100644 lib/gitlab/email/attachment_uploader.rb
 create mode 100644 lib/gitlab/email/html_cleaner.rb
 create mode 100644 lib/gitlab/email/receiver.rb
 create mode 100644 lib/gitlab/email/reply_parser.rb
 delete mode 100644 lib/gitlab/email_html_cleaner.rb
 delete mode 100644 lib/gitlab/email_receiver.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb
new file mode 100644
index 00000000000..0c0f50f2751
--- /dev/null
+++ b/lib/gitlab/email/attachment_uploader.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Email
+    module AttachmentUploader
+      attr_accessor :message
+
+      def initialize(message)
+        @message = message
+      end
+
+      def execute(project)
+        attachments = []
+
+        message.attachments.each do |attachment|
+          tmp = Tempfile.new("gitlab-email-attachment")
+          begin
+            File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded }
+
+            file = {
+              tempfile:     tmp,
+              filename:     attachment.filename,
+              content_type: attachment.content_type
+            }
+
+            link = ::Projects::UploadService.new(project, file).execute
+            attachments << link if link
+          ensure
+            tmp.close!
+          end
+        end
+
+        attachments
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/html_cleaner.rb b/lib/gitlab/email/html_cleaner.rb
new file mode 100644
index 00000000000..e1ae9eee56c
--- /dev/null
+++ b/lib/gitlab/email/html_cleaner.rb
@@ -0,0 +1,135 @@
+# Taken mostly from Discourse's Email::HtmlCleaner
+module Gitlab
+  module Email
+    # HtmlCleaner cleans up the extremely dirty HTML that many email clients
+    # generate by stripping out any excess divs or spans, removing styling in
+    # the process (which also makes the html more suitable to be parsed as
+    # Markdown).
+    class HtmlCleaner
+      # Elements to hoist all children out of
+      HTML_HOIST_ELEMENTS = %w(div span font table tbody th tr td)
+      # Node types to always delete
+      HTML_DELETE_ELEMENT_TYPES = [
+        Nokogiri::XML::Node::DTD_NODE,
+        Nokogiri::XML::Node::COMMENT_NODE,
+      ]
+
+      # Private variables:
+      #   @doc - nokogiri document
+      #   @out - same as @doc, but only if trimming has occured
+      def initialize(html)
+        if html.is_a?(String)
+          @doc = Nokogiri::HTML(html)
+        else
+          @doc = html
+        end
+      end
+
+      class << self
+        # HtmlCleaner.trim(inp, opts={})
+        #
+        # Arguments:
+        #   inp - Either a HTML string or a Nokogiri document.
+        # Options:
+        #   :return => :doc, :string
+        #     Specify the desired return type.
+        #     Defaults to the type of the input.
+        #     A value of :string is equivalent to calling get_document_text()
+        #     on the returned document.
+        def trim(inp, opts={})
+          cleaner = HtmlCleaner.new(inp)
+
+          opts[:return] ||= (inp.is_a?(String) ? :string : :doc)
+
+          if opts[:return] == :string
+            cleaner.output_html
+          else
+            cleaner.output_document
+          end
+        end
+
+        # HtmlCleaner.get_document_text(doc)
+        #
+        # Get the body portion of the document, including html, as a string.
+        def get_document_text(doc)
+          body = doc.xpath('//body')
+          if body
+            body.inner_html
+          else
+            doc.inner_html
+          end
+        end
+      end
+
+      def output_document
+        @out ||= begin
+          doc = @doc
+          trim_process_node doc
+          add_newlines doc
+          doc
+        end
+      end
+
+      def output_html
+        HtmlCleaner.get_document_text(output_document)
+      end
+
+      private
+
+      def add_newlines(doc)
+        # Replace <br> tags with a markdown \n
+        doc.xpath('//br').each do |br|
+          br.replace(new_linebreak_node doc, 2)
+        end
+        # Surround <p> tags with newlines, to help with line-wise postprocessing
+        # and ensure markdown paragraphs
+        doc.xpath('//p').each do |p|
+          p.before(new_linebreak_node doc)
+          p.after(new_linebreak_node doc, 2)
+        end
+      end
+
+      def new_linebreak_node(doc, count=1)
+        Nokogiri::XML::Text.new("\n" * count, doc)
+      end
+
+      def trim_process_node(node)
+        if should_hoist?(node)
+          hoisted = trim_hoist_element node
+          hoisted.each { |child| trim_process_node child }
+        elsif should_delete?(node)
+          node.remove
+        else
+          if children = node.children
+            children.each { |child| trim_process_node child }
+          end
+        end
+
+        node
+      end
+
+      def trim_hoist_element(element)
+        hoisted = []
+        element.children.each do |child|
+          element.before(child)
+          hoisted << child
+        end
+        element.remove
+        hoisted
+      end
+
+      def should_hoist?(node)
+        return false unless node.element?
+        HTML_HOIST_ELEMENTS.include? node.name
+      end
+
+      def should_delete?(node)
+        return true if HTML_DELETE_ELEMENT_TYPES.include? node.type
+        return true if node.element? && node.name == 'head'
+        return true if node.text? && node.text.strip.blank?
+
+        false
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
new file mode 100644
index 00000000000..c46fce6afe2
--- /dev/null
+++ b/lib/gitlab/email/receiver.rb
@@ -0,0 +1,101 @@
+# Inspired in great part by Discourse's Email::Receiver
+module Gitlab
+  module Email
+    class Receiver
+      class ProcessingError < StandardError; end
+      class EmailUnparsableError < ProcessingError; end
+      class EmptyEmailError < ProcessingError; end
+      class UserNotFoundError < ProcessingError; end
+      class UserNotAuthorizedError < ProcessingError; end
+      class NoteableNotFoundError < ProcessingError; end
+      class AutoGeneratedEmailError < ProcessingError; end
+      class SentNotificationNotFound < ProcessingError; end
+      class InvalidNote < ProcessingError; end
+
+      def initialize(raw)
+        @raw = raw
+      end
+
+      def message
+        @message ||= Mail::Message.new(@raw)
+      rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e
+        raise EmailUnparsableError, e
+      end
+
+      def execute
+        raise SentNotificationNotFound unless sent_notification
+
+        raise EmptyEmailError if @raw.blank?
+
+        raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/
+
+        author = sent_notification.recipient
+
+        raise UserNotFoundError unless author
+
+        project = sent_notification.project
+
+        raise UserNotAuthorizedError unless author.can?(:create_note, project)
+
+        raise NoteableNotFoundError unless sent_notification.noteable
+
+        reply = ReplyParser.new(message).execute.strip
+
+        raise EmptyEmailError if reply.blank?
+
+        reply = add_attachments(reply)
+
+        note = create_note(reply)
+
+        unless note.persisted?
+          message = "The comment could not be created for the following reasons:"
+          note.errors.full_messages.each do |error|
+            message << "\n\n- #{error}"
+          end
+
+          raise InvalidNote, message
+        end
+      end
+
+      private
+
+      def reply_key
+        reply_key = nil
+        message.to.each do |address|
+          reply_key = Gitlab::ReplyByEmail.reply_key_from_address(address)
+          break if reply_key
+        end
+
+        reply_key
+      end
+
+      def sent_notification
+        return nil unless reply_key
+        
+        SentNotification.for(reply_key)
+      end
+
+      def add_attachments(reply)
+        attachments = AttachmentUploader.new(message).execute(project)
+
+        attachments.each do |link|
+          text = "[#{link[:alt]}](#{link[:url]})"
+          text.prepend("!") if link[:is_image]
+
+          reply << "\n\n#{text}"
+        end
+      end
+
+      def create_note(reply)
+        Notes::CreateService.new(
+          sent_notification.project,
+          sent_notification.recipient,
+          note:           reply,
+          noteable_type:  sent_notification.noteable_type,
+          noteable_id:    sent_notification.noteable_id,
+          commit_id:      sent_notification.commit_id
+        ).execute
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb
new file mode 100644
index 00000000000..6ceb755968c
--- /dev/null
+++ b/lib/gitlab/email/reply_parser.rb
@@ -0,0 +1,91 @@
+# Inspired in great part by Discourse's Email::Receiver
+module Gitlab
+  module Email
+    class ReplyParser
+      attr_accessor :message
+
+      def initialize(message)
+        @message = message
+      end
+
+      def execute
+        body = select_body(message)
+
+        encoding = body.encoding
+
+        body = discourse_email_trimmer(body)
+
+        body = EmailReplyParser.parse_reply(body)
+
+        body.force_encoding(encoding).encode("UTF-8")
+      end
+
+      private
+
+      def select_body(message)
+        html = nil
+        text = nil
+
+        if message.multipart?
+          html = fix_charset(message.html_part)
+          text = fix_charset(message.text_part)
+        elsif message.content_type =~ /text\/html/
+          html = fix_charset(message)
+        end
+
+        # prefer plain text
+        return text if text
+
+        if html
+          body = HtmlCleaner.new(html).output_html
+        else
+          body = fix_charset(message)
+        end
+
+        # Certain trigger phrases that means we didn't parse correctly
+        if body =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/
+          return ""
+        end
+
+        body
+      end
+
+      # Force encoding to UTF-8 on a Mail::Message or Mail::Part
+      def fix_charset(object)
+        return nil if object.nil?
+
+        if object.charset
+          object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s
+        else
+          object.body.to_s
+        end
+      rescue
+        nil
+      end
+
+      REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date)
+      REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" })
+
+      def discourse_email_trimmer(body)
+        lines = body.scrub.lines.to_a
+        range_end = 0
+
+        lines.each_with_index do |l, idx|
+          # This one might be controversial but so many reply lines have years, times and end with a colon.
+         # Let's try it and see how well it works.
+         break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
+                   (l =~ /On \w+ \d+,? \d+,?.*wrote:/)
+
+          # Headers on subsequent lines
+          break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX }
+          # Headers on the same line
+          break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3
+
+          range_end = idx
+        end
+
+        lines[0..range_end].join.strip
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email_html_cleaner.rb b/lib/gitlab/email_html_cleaner.rb
deleted file mode 100644
index 6d7a17fe87c..00000000000
--- a/lib/gitlab/email_html_cleaner.rb
+++ /dev/null
@@ -1,133 +0,0 @@
-# Taken mostly from Discourse's Email::HtmlCleaner
-module Gitlab
-  # HtmlCleaner cleans up the extremely dirty HTML that many email clients
-  # generate by stripping out any excess divs or spans, removing styling in
-  # the process (which also makes the html more suitable to be parsed as
-  # Markdown).
-  class EmailHtmlCleaner
-    # Elements to hoist all children out of
-    HTML_HOIST_ELEMENTS = %w(div span font table tbody th tr td)
-    # Node types to always delete
-    HTML_DELETE_ELEMENT_TYPES = [
-      Nokogiri::XML::Node::DTD_NODE,
-      Nokogiri::XML::Node::COMMENT_NODE,
-    ]
-
-    # Private variables:
-    #   @doc - nokogiri document
-    #   @out - same as @doc, but only if trimming has occured
-    def initialize(html)
-      if html.is_a?(String)
-        @doc = Nokogiri::HTML(html)
-      else
-        @doc = html
-      end
-    end
-
-    class << self
-      # EmailHtmlCleaner.trim(inp, opts={})
-      #
-      # Arguments:
-      #   inp - Either a HTML string or a Nokogiri document.
-      # Options:
-      #   :return => :doc, :string
-      #     Specify the desired return type.
-      #     Defaults to the type of the input.
-      #     A value of :string is equivalent to calling get_document_text()
-      #     on the returned document.
-      def trim(inp, opts={})
-        cleaner = EmailHtmlCleaner.new(inp)
-
-        opts[:return] ||= (inp.is_a?(String) ? :string : :doc)
-
-        if opts[:return] == :string
-          cleaner.output_html
-        else
-          cleaner.output_document
-        end
-      end
-
-      # EmailHtmlCleaner.get_document_text(doc)
-      #
-      # Get the body portion of the document, including html, as a string.
-      def get_document_text(doc)
-        body = doc.xpath('//body')
-        if body
-          body.inner_html
-        else
-          doc.inner_html
-        end
-      end
-    end
-
-    def output_document
-      @out ||= begin
-        doc = @doc
-        trim_process_node doc
-        add_newlines doc
-        doc
-      end
-    end
-
-    def output_html
-      EmailHtmlCleaner.get_document_text(output_document)
-    end
-
-    private
-
-    def add_newlines(doc)
-      # Replace <br> tags with a markdown \n
-      doc.xpath('//br').each do |br|
-        br.replace(new_linebreak_node doc, 2)
-      end
-      # Surround <p> tags with newlines, to help with line-wise postprocessing
-      # and ensure markdown paragraphs
-      doc.xpath('//p').each do |p|
-        p.before(new_linebreak_node doc)
-        p.after(new_linebreak_node doc, 2)
-      end
-    end
-
-    def new_linebreak_node(doc, count=1)
-      Nokogiri::XML::Text.new("\n" * count, doc)
-    end
-
-    def trim_process_node(node)
-      if should_hoist?(node)
-        hoisted = trim_hoist_element node
-        hoisted.each { |child| trim_process_node child }
-      elsif should_delete?(node)
-        node.remove
-      else
-        if children = node.children
-          children.each { |child| trim_process_node child }
-        end
-      end
-
-      node
-    end
-
-    def trim_hoist_element(element)
-      hoisted = []
-      element.children.each do |child|
-        element.before(child)
-        hoisted << child
-      end
-      element.remove
-      hoisted
-    end
-
-    def should_hoist?(node)
-      return false unless node.element?
-      HTML_HOIST_ELEMENTS.include? node.name
-    end
-
-    def should_delete?(node)
-      return true if HTML_DELETE_ELEMENT_TYPES.include? node.type
-      return true if node.element? && node.name == 'head'
-      return true if node.text? && node.text.strip.blank?
-
-      false
-    end
-  end
-end
diff --git a/lib/gitlab/email_receiver.rb b/lib/gitlab/email_receiver.rb
deleted file mode 100644
index 3c1f346c0cf..00000000000
--- a/lib/gitlab/email_receiver.rb
+++ /dev/null
@@ -1,192 +0,0 @@
-# Inspired in great part by Discourse's Email::Receiver
-module Gitlab
-  class EmailReceiver
-    class ProcessingError < StandardError; end
-    class EmailUnparsableError < ProcessingError; end
-    class EmptyEmailError < ProcessingError; end
-    class UserNotFoundError < ProcessingError; end
-    class UserNotAuthorizedError < ProcessingError; end
-    class NoteableNotFoundError < ProcessingError; end
-    class AutoGeneratedEmailError < ProcessingError; end
-    class SentNotificationNotFound < ProcessingError; end
-    class InvalidNote < ProcessingError; end
-
-    def initialize(raw)
-      @raw = raw
-    end
-
-    def message
-      @message ||= Mail::Message.new(@raw)
-    rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e
-      raise EmailUnparsableError, e
-    end
-
-    def execute
-      raise SentNotificationNotFound unless sent_notification
-
-      raise EmptyEmailError if @raw.blank?
-
-      raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/
-
-      author = sent_notification.recipient
-
-      raise UserNotFoundError unless author
-
-      project = sent_notification.project
-
-      raise UserNotAuthorizedError unless author.can?(:create_note, project)
-
-      raise NoteableNotFoundError unless sent_notification.noteable
-
-      body = parse_body(message)
-
-      upload_attachments.each do |link|
-        body << "\n\n#{link}"
-      end
-
-      note = Notes::CreateService.new(
-        project,
-        author,
-        note:           body,
-        noteable_type:  sent_notification.noteable_type,
-        noteable_id:    sent_notification.noteable_id,
-        commit_id:      sent_notification.commit_id
-      ).execute
-
-      unless note.persisted?
-        message = "The comment could not be created for the following reasons:"
-        note.errors.full_messages.each do |error|
-          message << "\n\n- #{error}"
-        end
-        raise InvalidNote, message
-      end
-    end
-
-    def parse_body(message)
-      body = select_body(message)
-
-      encoding = body.encoding
-      raise EmptyEmailError if body.strip.blank?
-
-      body = discourse_email_trimmer(body)
-      raise EmptyEmailError if body.strip.blank?
-
-      body = EmailReplyParser.parse_reply(body)
-      raise EmptyEmailError if body.strip.blank?
-
-      body.force_encoding(encoding).encode("UTF-8")
-    end
-
-    private
-
-    def reply_key
-      reply_key = nil
-      message.to.each do |address|
-        reply_key = Gitlab::ReplyByEmail.reply_key_from_address(address)
-        break if reply_key
-      end
-
-      reply_key
-    end
-
-    def sent_notification
-      return nil unless reply_key
-      
-      SentNotification.for(reply_key)
-    end
-
-    def select_body(message)
-      html = nil
-      text = nil
-
-      if message.multipart?
-        html = fix_charset(message.html_part)
-        text = fix_charset(message.text_part)
-      elsif message.content_type =~ /text\/html/
-        html = fix_charset(message)
-      end
-
-      # prefer plain text
-      return text if text
-
-      if html
-        body = EmailHtmlCleaner.new(html).output_html
-      else
-        body = fix_charset(message)
-      end
-
-      # Certain trigger phrases that means we didn't parse correctly
-      if body =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/
-        raise EmptyEmailError
-      end
-
-      body
-    end
-
-    # Force encoding to UTF-8 on a Mail::Message or Mail::Part
-    def fix_charset(object)
-      return nil if object.nil?
-
-      if object.charset
-        object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s
-      else
-        object.body.to_s
-      end
-    rescue
-      nil
-    end
-
-    REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date)
-    REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" })
-
-    def discourse_email_trimmer(body)
-      lines = body.scrub.lines.to_a
-      range_end = 0
-
-      lines.each_with_index do |l, idx|
-        # This one might be controversial but so many reply lines have years, times and end with a colon.
-       # Let's try it and see how well it works.
-       break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
-                 (l =~ /On \w+ \d+,? \d+,?.*wrote:/)
-
-        # Headers on subsequent lines
-        break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX }
-        # Headers on the same line
-        break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3
-
-        range_end = idx
-      end
-
-      lines[0..range_end].join.strip
-    end
-
-    def upload_attachments
-      attachments = []
-
-      message.attachments.each do |attachment|
-        tmp = Tempfile.new("gitlab-email-attachment")
-        begin
-          File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded }
-
-          file = {
-            tempfile:     tmp,
-            filename:     attachment.filename,
-            content_type: attachment.content_type
-          }
-
-          link = ::Projects::UploadService.new(sent_notification.project, file).execute
-          if link
-            text = "[#{link[:alt]}](#{link[:url]})"
-            text.prepend("!") if link[:is_image]
-
-            attachments << text
-          end
-        ensure
-          tmp.close!
-        end
-      end
-
-      attachments
-    end
-  end
-end
-- 
cgit v1.2.3


From 0b401f2e94c5062d26887cdeda73341c1aae82a5 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Thu, 20 Aug 2015 11:17:14 -0700
Subject: Fix a couple of whoopsy daisies.

---
 lib/gitlab/email/attachment_uploader.rb | 2 +-
 lib/gitlab/email/receiver.rb            | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb
index 0c0f50f2751..32cece8316b 100644
--- a/lib/gitlab/email/attachment_uploader.rb
+++ b/lib/gitlab/email/attachment_uploader.rb
@@ -1,6 +1,6 @@
 module Gitlab
   module Email
-    module AttachmentUploader
+    class AttachmentUploader
       attr_accessor :message
 
       def initialize(message)
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index c46fce6afe2..7160af29089 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -76,7 +76,7 @@ module Gitlab
       end
 
       def add_attachments(reply)
-        attachments = AttachmentUploader.new(message).execute(project)
+        attachments = Email::AttachmentUploader.new(message).execute(sent_notification.project)
 
         attachments.each do |link|
           text = "[#{link[:alt]}](#{link[:url]})"
@@ -84,6 +84,8 @@ module Gitlab
 
           reply << "\n\n#{text}"
         end
+
+        reply
       end
 
       def create_note(reply)
-- 
cgit v1.2.3


From 2f78b5e8afbf0024211bbd5bfe3a6c6f53b2421e Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Thu, 20 Aug 2015 11:33:18 -0700
Subject: Make error class names more consistent.

---
 lib/gitlab/email/receiver.rb | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 7160af29089..466e48b22e1 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -4,13 +4,13 @@ module Gitlab
     class Receiver
       class ProcessingError < StandardError; end
       class EmailUnparsableError < ProcessingError; end
+      class SentNotificationNotFoundError < ProcessingError; end
       class EmptyEmailError < ProcessingError; end
+      class AutoGeneratedEmailError < ProcessingError; end
       class UserNotFoundError < ProcessingError; end
       class UserNotAuthorizedError < ProcessingError; end
       class NoteableNotFoundError < ProcessingError; end
-      class AutoGeneratedEmailError < ProcessingError; end
-      class SentNotificationNotFound < ProcessingError; end
-      class InvalidNote < ProcessingError; end
+      class InvalidNoteError < ProcessingError; end
 
       def initialize(raw)
         @raw = raw
@@ -23,7 +23,7 @@ module Gitlab
       end
 
       def execute
-        raise SentNotificationNotFound unless sent_notification
+        raise SentNotificationNotFoundError unless sent_notification
 
         raise EmptyEmailError if @raw.blank?
 
@@ -53,7 +53,7 @@ module Gitlab
             message << "\n\n- #{error}"
           end
 
-          raise InvalidNote, message
+          raise InvalidNoteError, message
         end
       end
 
-- 
cgit v1.2.3


From 8ec5fb138dde9937814ac138352177399d3e776d Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Thu, 20 Aug 2015 12:17:59 -0700
Subject: Test Gitlab::Email::Receiver.

---
 lib/gitlab/email/receiver.rb | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 466e48b22e1..17b8339edcd 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -16,17 +16,11 @@ module Gitlab
         @raw = raw
       end
 
-      def message
-        @message ||= Mail::Message.new(@raw)
-      rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e
-        raise EmailUnparsableError, e
-      end
-
       def execute
-        raise SentNotificationNotFoundError unless sent_notification
-
         raise EmptyEmailError if @raw.blank?
 
+        raise SentNotificationNotFoundError unless sent_notification
+
         raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/
 
         author = sent_notification.recipient
@@ -35,7 +29,7 @@ module Gitlab
 
         project = sent_notification.project
 
-        raise UserNotAuthorizedError unless author.can?(:create_note, project)
+        raise UserNotAuthorizedError unless project && author.can?(:create_note, project)
 
         raise NoteableNotFoundError unless sent_notification.noteable
 
@@ -59,6 +53,12 @@ module Gitlab
 
       private
 
+      def message
+        @message ||= Mail::Message.new(@raw)
+      rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e
+        raise EmailUnparsableError, e
+      end
+
       def reply_key
         reply_key = nil
         message.to.each do |address|
-- 
cgit v1.2.3


From 123af7856167f01ac0a7873bca1ef2cbb835e3b5 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Thu, 20 Aug 2015 14:03:04 -0700
Subject: Add gitlab:reply_by_email:check rake task.

---
 lib/gitlab/reply_by_email.rb | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/reply_by_email.rb b/lib/gitlab/reply_by_email.rb
index b6157de3610..61d048fd200 100644
--- a/lib/gitlab/reply_by_email.rb
+++ b/lib/gitlab/reply_by_email.rb
@@ -2,9 +2,12 @@ module Gitlab
   module ReplyByEmail
     class << self
       def enabled?
-        config.enabled &&
-          config.address &&
-          config.address.include?("%{reply_key}")
+        config.enabled && address_formatted_correctly?
+      end
+
+      def address_formatted_correctly?
+        config.address &&
+        config.address.include?("%{reply_key}")
       end
 
       def reply_key
-- 
cgit v1.2.3


From f26c2905d198436ea4c4b7f26c6971edd4ff8049 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Thu, 20 Aug 2015 14:25:56 -0700
Subject: Fix indentation

---
 lib/gitlab/email/reply_parser.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb
index 6ceb755968c..6e768e46a71 100644
--- a/lib/gitlab/email/reply_parser.rb
+++ b/lib/gitlab/email/reply_parser.rb
@@ -72,8 +72,8 @@ module Gitlab
 
         lines.each_with_index do |l, idx|
           # This one might be controversial but so many reply lines have years, times and end with a colon.
-         # Let's try it and see how well it works.
-         break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
+          # Let's try it and see how well it works.
+          break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
                    (l =~ /On \w+ \d+,? \d+,?.*wrote:/)
 
           # Headers on subsequent lines
-- 
cgit v1.2.3


From 99ef8c81598ad31922dfbe28c0c56130e01bd13a Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Thu, 20 Aug 2015 15:37:43 -0700
Subject: Fix markdown specs.

---
 lib/gitlab/reply_by_email.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/reply_by_email.rb b/lib/gitlab/reply_by_email.rb
index 61d048fd200..4fdeb6b8a74 100644
--- a/lib/gitlab/reply_by_email.rb
+++ b/lib/gitlab/reply_by_email.rb
@@ -7,7 +7,7 @@ module Gitlab
 
       def address_formatted_correctly?
         config.address &&
-        config.address.include?("%{reply_key}")
+          config.address.include?("%{reply_key}")
       end
 
       def reply_key
-- 
cgit v1.2.3


From 3d141d1512a39b61159026457b31c6f0aecf5b6c Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Thu, 20 Aug 2015 16:21:31 -0700
Subject: Fix spec by removing global state.

---
 lib/gitlab/reply_by_email.rb | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/reply_by_email.rb b/lib/gitlab/reply_by_email.rb
index 4fdeb6b8a74..f93fda4302c 100644
--- a/lib/gitlab/reply_by_email.rb
+++ b/lib/gitlab/reply_by_email.rb
@@ -36,14 +36,12 @@ module Gitlab
       end
 
       def address_regex
-        @address_regex ||= begin
-          wildcard_address = config.address
-          return nil unless wildcard_address
-
-          regex = Regexp.escape(wildcard_address)
-          regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.+)")
-          Regexp.new(regex).freeze
-        end
+        wildcard_address = config.address
+        return nil unless wildcard_address
+
+        regex = Regexp.escape(wildcard_address)
+        regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.+)")
+        Regexp.new(regex).freeze
       end
     end
   end
-- 
cgit v1.2.3


From 35224d5e7f3e0c978640b7a6dd64e9778c4d1c60 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Thu, 20 Aug 2015 16:44:44 -0700
Subject: Memoize address_regex locally.

---
 lib/gitlab/reply_by_email.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/reply_by_email.rb b/lib/gitlab/reply_by_email.rb
index f93fda4302c..c3fe6778f06 100644
--- a/lib/gitlab/reply_by_email.rb
+++ b/lib/gitlab/reply_by_email.rb
@@ -21,9 +21,10 @@ module Gitlab
       end
 
       def reply_key_from_address(address)
-        return unless address_regex
+        regex = address_regex
+        return unless regex
 
-        match = address.match(address_regex)
+        match = address.match(regex)
         return unless match
 
         match[1]
-- 
cgit v1.2.3


From 747fe7520b244a324b60049cbe22c22a5df29c82 Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Thu, 20 Aug 2015 18:25:16 -0700
Subject: Remove trailing HTML entities from non-Rinku autolinks as well.

---
 lib/gitlab/markdown/autolink_filter.rb | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/markdown/autolink_filter.rb b/lib/gitlab/markdown/autolink_filter.rb
index 4e14a048cfb..541f1d88ffc 100644
--- a/lib/gitlab/markdown/autolink_filter.rb
+++ b/lib/gitlab/markdown/autolink_filter.rb
@@ -87,8 +87,14 @@ module Gitlab
 
       def autolink_filter(text)
         text.gsub(LINK_PATTERN) do |match|
+          # Remove any trailing HTML entities and store them for appending
+          # outside the link element. The entity must be marked HTML safe in
+          # order to be output literally rather than escaped.
+          match.gsub!(/((?:&[\w#]+;)+)\z/, '')
+          dropped = ($1 || '').html_safe
+
           options = link_options.merge(href: match)
-          content_tag(:a, match, options)
+          content_tag(:a, match, options) + dropped
         end
       end
 
-- 
cgit v1.2.3


From 69708dab9f6e1c265dd2bf80eafc39bf68c356e0 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Fri, 21 Aug 2015 10:14:45 -0700
Subject: Block blocked users from replying to threads by email.

---
 lib/gitlab/email/receiver.rb | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
index 17b8339edcd..355fbd27898 100644
--- a/lib/gitlab/email/receiver.rb
+++ b/lib/gitlab/email/receiver.rb
@@ -8,6 +8,7 @@ module Gitlab
       class EmptyEmailError < ProcessingError; end
       class AutoGeneratedEmailError < ProcessingError; end
       class UserNotFoundError < ProcessingError; end
+      class UserBlockedError < ProcessingError; end
       class UserNotAuthorizedError < ProcessingError; end
       class NoteableNotFoundError < ProcessingError; end
       class InvalidNoteError < ProcessingError; end
@@ -27,6 +28,8 @@ module Gitlab
 
         raise UserNotFoundError unless author
 
+        raise UserBlockedError if author.blocked?
+
         project = sent_notification.project
 
         raise UserNotAuthorizedError unless project && author.can?(:create_note, project)
-- 
cgit v1.2.3


From 15fc7bd6139f0b429c05c055b4cfab561c926e08 Mon Sep 17 00:00:00 2001
From: Douwe Maan <douwe@gitlab.com>
Date: Fri, 21 Aug 2015 16:09:55 -0700
Subject: No HTML-only email please

---
 lib/gitlab/email/html_cleaner.rb | 135 ---------------------------------------
 lib/gitlab/email/reply_parser.rb |  24 ++-----
 2 files changed, 6 insertions(+), 153 deletions(-)
 delete mode 100644 lib/gitlab/email/html_cleaner.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/email/html_cleaner.rb b/lib/gitlab/email/html_cleaner.rb
deleted file mode 100644
index e1ae9eee56c..00000000000
--- a/lib/gitlab/email/html_cleaner.rb
+++ /dev/null
@@ -1,135 +0,0 @@
-# Taken mostly from Discourse's Email::HtmlCleaner
-module Gitlab
-  module Email
-    # HtmlCleaner cleans up the extremely dirty HTML that many email clients
-    # generate by stripping out any excess divs or spans, removing styling in
-    # the process (which also makes the html more suitable to be parsed as
-    # Markdown).
-    class HtmlCleaner
-      # Elements to hoist all children out of
-      HTML_HOIST_ELEMENTS = %w(div span font table tbody th tr td)
-      # Node types to always delete
-      HTML_DELETE_ELEMENT_TYPES = [
-        Nokogiri::XML::Node::DTD_NODE,
-        Nokogiri::XML::Node::COMMENT_NODE,
-      ]
-
-      # Private variables:
-      #   @doc - nokogiri document
-      #   @out - same as @doc, but only if trimming has occured
-      def initialize(html)
-        if html.is_a?(String)
-          @doc = Nokogiri::HTML(html)
-        else
-          @doc = html
-        end
-      end
-
-      class << self
-        # HtmlCleaner.trim(inp, opts={})
-        #
-        # Arguments:
-        #   inp - Either a HTML string or a Nokogiri document.
-        # Options:
-        #   :return => :doc, :string
-        #     Specify the desired return type.
-        #     Defaults to the type of the input.
-        #     A value of :string is equivalent to calling get_document_text()
-        #     on the returned document.
-        def trim(inp, opts={})
-          cleaner = HtmlCleaner.new(inp)
-
-          opts[:return] ||= (inp.is_a?(String) ? :string : :doc)
-
-          if opts[:return] == :string
-            cleaner.output_html
-          else
-            cleaner.output_document
-          end
-        end
-
-        # HtmlCleaner.get_document_text(doc)
-        #
-        # Get the body portion of the document, including html, as a string.
-        def get_document_text(doc)
-          body = doc.xpath('//body')
-          if body
-            body.inner_html
-          else
-            doc.inner_html
-          end
-        end
-      end
-
-      def output_document
-        @out ||= begin
-          doc = @doc
-          trim_process_node doc
-          add_newlines doc
-          doc
-        end
-      end
-
-      def output_html
-        HtmlCleaner.get_document_text(output_document)
-      end
-
-      private
-
-      def add_newlines(doc)
-        # Replace <br> tags with a markdown \n
-        doc.xpath('//br').each do |br|
-          br.replace(new_linebreak_node doc, 2)
-        end
-        # Surround <p> tags with newlines, to help with line-wise postprocessing
-        # and ensure markdown paragraphs
-        doc.xpath('//p').each do |p|
-          p.before(new_linebreak_node doc)
-          p.after(new_linebreak_node doc, 2)
-        end
-      end
-
-      def new_linebreak_node(doc, count=1)
-        Nokogiri::XML::Text.new("\n" * count, doc)
-      end
-
-      def trim_process_node(node)
-        if should_hoist?(node)
-          hoisted = trim_hoist_element node
-          hoisted.each { |child| trim_process_node child }
-        elsif should_delete?(node)
-          node.remove
-        else
-          if children = node.children
-            children.each { |child| trim_process_node child }
-          end
-        end
-
-        node
-      end
-
-      def trim_hoist_element(element)
-        hoisted = []
-        element.children.each do |child|
-          element.before(child)
-          hoisted << child
-        end
-        element.remove
-        hoisted
-      end
-
-      def should_hoist?(node)
-        return false unless node.element?
-        HTML_HOIST_ELEMENTS.include? node.name
-      end
-
-      def should_delete?(node)
-        return true if HTML_DELETE_ELEMENT_TYPES.include? node.type
-        return true if node.element? && node.name == 'head'
-        return true if node.text? && node.text.strip.blank?
-
-        false
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb
index 6e768e46a71..6ed36b51f12 100644
--- a/lib/gitlab/email/reply_parser.rb
+++ b/lib/gitlab/email/reply_parser.rb
@@ -23,31 +23,19 @@ module Gitlab
       private
 
       def select_body(message)
-        html = nil
-        text = nil
-
-        if message.multipart?
-          html = fix_charset(message.html_part)
-          text = fix_charset(message.text_part)
-        elsif message.content_type =~ /text\/html/
-          html = fix_charset(message)
-        end
+        text    = message.text_part if message.multipart?
+        text  ||= message           if message.content_type !~ /text\/html/
 
-        # prefer plain text
-        return text if text
+        return "" unless text
 
-        if html
-          body = HtmlCleaner.new(html).output_html
-        else
-          body = fix_charset(message)
-        end
+        text = fix_charset(text)
 
         # Certain trigger phrases that means we didn't parse correctly
-        if body =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/
+        if text =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/
           return ""
         end
 
-        body
+        text
       end
 
       # Force encoding to UTF-8 on a Mail::Message or Mail::Part
-- 
cgit v1.2.3


From ed1d4fa477789659f9343593bf06d50e70750561 Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Fri, 7 Aug 2015 00:06:20 -0700
Subject: Remove user OAuth tokens stored in database for Bitbucket, GitHub,
 and GitLab and request them each session. Pass these tokens to the project
 import data.

This prevents the need to encrypt these tokens and clear them in case they
expire or get revoked.

For example, if you deleted and re-created OAuth2 keys for Bitbucket, you would get
an Error 500 with no way to recover:

```
Started GET "/import/bitbucket/status" for x.x.x.x at 2015-08-07 05:24:10 +0000
Processing by Import::BitbucketController#status as HTML
Completed 500 Internal Server Error in 607ms (ActiveRecord: 2.3ms)

NameError (uninitialized constant Import::BitbucketController::Unauthorized):
  app/controllers/import/bitbucket_controller.rb:77:in `rescue in go_to_bitbucket_for_permissions'
  app/controllers/import/bitbucket_controller.rb:74:in `go_to_bitbucket_for_permissions'
  app/controllers/import/bitbucket_controller.rb:86:in `bitbucket_unauthorized'
```

Closes #1871
---
 lib/gitlab/bitbucket_import/importer.rb        | 15 +++++++++------
 lib/gitlab/bitbucket_import/key_adder.rb       |  7 ++++---
 lib/gitlab/bitbucket_import/key_deleter.rb     |  7 +++++--
 lib/gitlab/bitbucket_import/project_creator.rb | 12 ++++++++----
 lib/gitlab/github_import/importer.rb           |  4 +++-
 lib/gitlab/github_import/project_creator.rb    | 13 +++++++++----
 lib/gitlab/gitlab_import/importer.rb           | 14 ++++++++------
 lib/gitlab/gitlab_import/project_creator.rb    | 12 ++++++++----
 8 files changed, 54 insertions(+), 30 deletions(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index 42c93707caa..d8a7d29f1bf 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -5,7 +5,10 @@ module Gitlab
 
       def initialize(project)
         @project = project
-        @client = Client.new(project.creator.bitbucket_access_token, project.creator.bitbucket_access_token_secret)
+        import_data = project.import_data.try(:data)
+        bb_session = import_data["bb_session"] if import_data
+        @client = Client.new(bb_session["bitbucket_access_token"],
+                             bb_session["bitbucket_access_token_secret"])
         @formatter = Gitlab::ImportFormatter.new
       end
 
@@ -16,12 +19,12 @@ module Gitlab
 
         #Issues && Comments
         issues = client.issues(project_identifier)
-        
+
         issues["issues"].each do |issue|
           body = @formatter.author_line(issue["reported_by"]["username"], issue["content"])
-          
+
           comments = client.issue_comments(project_identifier, issue["local_id"])
-          
+
           if comments.any?
             body += @formatter.comments_header
           end
@@ -31,13 +34,13 @@ module Gitlab
           end
 
           project.issues.create!(
-            description: body, 
+            description: body,
             title: issue["title"],
             state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened',
             author_id: gl_user_id(project, issue["reported_by"]["username"])
           )
         end
-        
+
         true
       end
 
diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb
index 9931aa7e029..0b63f025d0a 100644
--- a/lib/gitlab/bitbucket_import/key_adder.rb
+++ b/lib/gitlab/bitbucket_import/key_adder.rb
@@ -3,14 +3,15 @@ module Gitlab
     class KeyAdder
       attr_reader :repo, :current_user, :client
 
-      def initialize(repo, current_user)
+      def initialize(repo, current_user, access_params)
         @repo, @current_user = repo, current_user
-        @client = Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret)
+        @client = Client.new(access_params[:bitbucket_access_token],
+                             access_params[:bitbucket_access_token_secret])
       end
 
       def execute
         return false unless BitbucketImport.public_key.present?
-        
+
         project_identifier = "#{repo["owner"]}/#{repo["slug"]}"
         client.add_deploy_key(project_identifier, BitbucketImport.public_key)
 
diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb
index 1a24a86fc37..f4dd393ad29 100644
--- a/lib/gitlab/bitbucket_import/key_deleter.rb
+++ b/lib/gitlab/bitbucket_import/key_deleter.rb
@@ -6,12 +6,15 @@ module Gitlab
       def initialize(project)
         @project = project
         @current_user = project.creator
-        @client = Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret)
+        import_data = project.import_data.try(:data)
+        bb_session = import_data["bb_session"] if import_data
+        @client = Client.new(bb_session["bitbucket_access_token"],
+                             bb_session["bitbucket_access_token_secret"])
       end
 
       def execute
         return false unless BitbucketImport.public_key.present?
-        
+
         client.delete_deploy_key(project.import_source, BitbucketImport.public_key)
 
         true
diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb
index 54420e62c90..35e34d033e0 100644
--- a/lib/gitlab/bitbucket_import/project_creator.rb
+++ b/lib/gitlab/bitbucket_import/project_creator.rb
@@ -1,16 +1,17 @@
 module Gitlab
   module BitbucketImport
     class ProjectCreator
-      attr_reader :repo, :namespace, :current_user
+      attr_reader :repo, :namespace, :current_user, :session_data
 
-      def initialize(repo, namespace, current_user)
+      def initialize(repo, namespace, current_user, session_data)
         @repo = repo
         @namespace = namespace
         @current_user = current_user
+        @session_data = session_data
       end
 
       def execute
-        ::Projects::CreateService.new(current_user,
+        project = ::Projects::CreateService.new(current_user,
           name: repo["name"],
           path: repo["slug"],
           description: repo["description"],
@@ -18,8 +19,11 @@ module Gitlab
           visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
           import_type: "bitbucket",
           import_source: "#{repo["owner"]}/#{repo["slug"]}",
-          import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git"
+          import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git",
         ).execute
+
+        project.create_import_data(data: { "bb_session" => session_data } )
+        project
       end
     end
   end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 98039a76dcd..8c106a61735 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -5,7 +5,9 @@ module Gitlab
 
       def initialize(project)
         @project = project
-        @client = Client.new(project.creator.github_access_token)
+        import_data = project.import_data.try(:data)
+        github_session = import_data["github_session"] if import_data
+        @client = Client.new(github_session["github_access_token"])
         @formatter = Gitlab::ImportFormatter.new
       end
 
diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb
index 2723eec933e..8c27ebd1ce8 100644
--- a/lib/gitlab/github_import/project_creator.rb
+++ b/lib/gitlab/github_import/project_creator.rb
@@ -1,16 +1,18 @@
 module Gitlab
   module GithubImport
     class ProjectCreator
-      attr_reader :repo, :namespace, :current_user
+      attr_reader :repo, :namespace, :current_user, :session_data
 
-      def initialize(repo, namespace, current_user)
+      def initialize(repo, namespace, current_user, session_data)
         @repo = repo
         @namespace = namespace
         @current_user = current_user
+        @session_data = session_data
       end
 
       def execute
-        ::Projects::CreateService.new(current_user,
+        project = ::Projects::CreateService.new(
+          current_user,
           name: repo.name,
           path: repo.name,
           description: repo.description,
@@ -18,8 +20,11 @@ module Gitlab
           visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
           import_type: "github",
           import_source: repo.full_name,
-          import_url: repo.clone_url.sub("https://", "https://#{current_user.github_access_token}@")
+          import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@")
         ).execute
+
+        project.create_import_data(data: { "github_session" => session_data } )
+        project
       end
     end
   end
diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb
index c5304a0699b..50594d2b24f 100644
--- a/lib/gitlab/gitlab_import/importer.rb
+++ b/lib/gitlab/gitlab_import/importer.rb
@@ -5,7 +5,9 @@ module Gitlab
 
       def initialize(project)
         @project = project
-        @client = Client.new(project.creator.gitlab_access_token)
+        import_data = project.import_data.try(:data)
+        gitlab_session = import_data["gitlab_session"] if import_data
+        @client = Client.new(gitlab_session["gitlab_access_token"])
         @formatter = Gitlab::ImportFormatter.new
       end
 
@@ -14,12 +16,12 @@ module Gitlab
 
         #Issues && Comments
         issues = client.issues(project_identifier)
-        
+
         issues.each do |issue|
           body = @formatter.author_line(issue["author"]["name"], issue["description"])
-          
+
           comments = client.issue_comments(project_identifier, issue["id"])
-          
+
           if comments.any?
             body += @formatter.comments_header
           end
@@ -29,13 +31,13 @@ module Gitlab
           end
 
           project.issues.create!(
-            description: body, 
+            description: body,
             title: issue["title"],
             state: issue["state"],
             author_id: gl_user_id(project, issue["author"]["id"])
           )
         end
-        
+
         true
       end
 
diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb
index f0d7141bf56..d9452de6a50 100644
--- a/lib/gitlab/gitlab_import/project_creator.rb
+++ b/lib/gitlab/gitlab_import/project_creator.rb
@@ -1,16 +1,17 @@
 module Gitlab
   module GitlabImport
     class ProjectCreator
-      attr_reader :repo, :namespace, :current_user
+      attr_reader :repo, :namespace, :current_user, :session_data
 
-      def initialize(repo, namespace, current_user)
+      def initialize(repo, namespace, current_user, session_data)
         @repo = repo
         @namespace = namespace
         @current_user = current_user
+        @session_data = session_data
       end
 
       def execute
-        ::Projects::CreateService.new(current_user,
+        project = ::Projects::CreateService.new(current_user,
           name: repo["name"],
           path: repo["path"],
           description: repo["description"],
@@ -18,8 +19,11 @@ module Gitlab
           visibility_level: repo["visibility_level"],
           import_type: "gitlab",
           import_source: repo["path_with_namespace"],
-          import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{current_user.gitlab_access_token}@")
+          import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@")
         ).execute
+
+        project.create_import_data(data: { "gitlab_session" => session_data } )
+        project
       end
     end
   end
-- 
cgit v1.2.3


From 56527b63e8a09e0fe4967eabf08638d853e6b2b5 Mon Sep 17 00:00:00 2001
From: Valery Sizov <vsv2711@gmail.com>
Date: Thu, 13 Aug 2015 17:48:21 +0300
Subject: Ability to search milestones

---
 lib/gitlab/search_results.rb | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 06245374bc8..2ab2d4af797 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -19,13 +19,15 @@ module Gitlab
         issues.page(page).per(per_page)
       when 'merge_requests'
         merge_requests.page(page).per(per_page)
+      when 'milestones'
+        milestones.page(page).per(per_page)
       else
         Kaminari.paginate_array([]).page(page).per(per_page)
       end
     end
 
     def total_count
-      @total_count ||= projects_count + issues_count + merge_requests_count
+      @total_count ||= projects_count + issues_count + merge_requests_count + milestones_count
     end
 
     def projects_count
@@ -40,6 +42,10 @@ module Gitlab
       @merge_requests_count ||= merge_requests.count
     end
 
+    def milestones_count
+      @milestones_count ||= milestones.count
+    end
+
     def empty?
       total_count.zero?
     end
@@ -60,6 +66,12 @@ module Gitlab
       issues.order('updated_at DESC')
     end
 
+    def milestones
+      milestones = Milestone.where(project_id: limit_project_ids)
+      milestones = milestones.search(query)
+      milestones.order('updated_at DESC')
+    end
+
     def merge_requests
       merge_requests = MergeRequest.in_projects(limit_project_ids)
       if query =~ /[#!](\d+)\z/
-- 
cgit v1.2.3


From 4344b8d2d4367b19c6849c3cab0d02d17ddf2304 Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Tue, 25 Aug 2015 14:31:33 -0700
Subject: Add Gitlab::ColorSchemes module

Very similar to Gitlab::Theme, this contains all of the definitions for
our syntax highlighting schemes.
---
 lib/gitlab/color_schemes.rb | 62 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 62 insertions(+)
 create mode 100644 lib/gitlab/color_schemes.rb

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/color_schemes.rb b/lib/gitlab/color_schemes.rb
new file mode 100644
index 00000000000..763853ab1cb
--- /dev/null
+++ b/lib/gitlab/color_schemes.rb
@@ -0,0 +1,62 @@
+module Gitlab
+  # Module containing GitLab's syntax color scheme definitions and helper
+  # methods for accessing them.
+  module ColorSchemes
+    # Struct class representing a single Scheme
+    Scheme = Struct.new(:id, :name, :css_class)
+
+    SCHEMES = [
+      Scheme.new(1, 'White',           'white'),
+      Scheme.new(2, 'Dark',            'dark'),
+      Scheme.new(3, 'Solarized Light', 'solarized-light'),
+      Scheme.new(4, 'Solarized Dark',  'solarized-dark'),
+      Scheme.new(5, 'Monokai',         'monokai')
+    ].freeze
+
+    # Convenience method to get a space-separated String of all the color scheme
+    # classes that might be applied to a code block.
+    #
+    # Returns a String
+    def self.body_classes
+      SCHEMES.collect(&:css_class).uniq.join(' ')
+    end
+
+    # Get a Scheme by its ID
+    #
+    # If the ID is invalid, returns the default Scheme.
+    #
+    # id - Integer ID
+    #
+    # Returns a Scheme
+    def self.by_id(id)
+      SCHEMES.detect { |s| s.id == id } || default
+    end
+
+    # Get the default Scheme
+    #
+    # Returns a Scheme
+    def self.default
+      by_id(1)
+    end
+
+    # Iterate through each Scheme
+    #
+    # Yields the Scheme object
+    def self.each(&block)
+      SCHEMES.each(&block)
+    end
+
+    # Get the Scheme for the specified user, or the default
+    #
+    # user - User record
+    #
+    # Returns a Scheme
+    def self.for_user(user)
+      if user
+        by_id(user.color_scheme_id)
+      else
+        default
+      end
+    end
+  end
+end
-- 
cgit v1.2.3


From 6d43308b5a5ff7663e1782fd6f5493f48c0f7e2a Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Wed, 26 Aug 2015 11:30:11 -0700
Subject: Add `Gitlab::Themes.for_user`

---
 lib/gitlab/themes.rb | 13 +++++++++++++
 1 file changed, 13 insertions(+)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index 5209df92795..37a36b9599b 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -51,6 +51,19 @@ module Gitlab
       THEMES.each(&block)
     end
 
+    # Get the Theme for the specified user, or the default
+    #
+    # user - User record
+    #
+    # Returns a Theme
+    def self.for_user(user)
+      if user
+        by_id(user.theme_id)
+      else
+        default
+      end
+    end
+
     private
 
     def self.default_id
-- 
cgit v1.2.3


From 2d72efcd9fb63f4bb0c6aed5a472a2eca2d6abac Mon Sep 17 00:00:00 2001
From: Robert Speicher <rspeicher@gmail.com>
Date: Wed, 26 Aug 2015 11:30:38 -0700
Subject: Add `count` to Themes and ColorSchemes

---
 lib/gitlab/color_schemes.rb | 5 +++++
 lib/gitlab/themes.rb        | 5 +++++
 2 files changed, 10 insertions(+)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/color_schemes.rb b/lib/gitlab/color_schemes.rb
index 763853ab1cb..9c4664df903 100644
--- a/lib/gitlab/color_schemes.rb
+++ b/lib/gitlab/color_schemes.rb
@@ -32,6 +32,11 @@ module Gitlab
       SCHEMES.detect { |s| s.id == id } || default
     end
 
+    # Returns the number of defined Schemes
+    def self.count
+      SCHEMES.size
+    end
+
     # Get the default Scheme
     #
     # Returns a Scheme
diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index 37a36b9599b..83f91de810c 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -37,6 +37,11 @@ module Gitlab
       THEMES.detect { |t| t.id == id } || default
     end
 
+    # Returns the number of defined Themes
+    def self.count
+      THEMES.size
+    end
+
     # Get the default Theme
     #
     # Returns a Theme
-- 
cgit v1.2.3


From 23aee0ca8ad3de5697f770696f3e55fbfdba2be8 Mon Sep 17 00:00:00 2001
From: Eric Maziade <eric@spoutnik.co>
Date: Wed, 26 Aug 2015 22:28:24 -0400
Subject: fixed connection detection so settings can be read from the database

---
 lib/gitlab/current_settings.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib/gitlab')

diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 1a2a50a14d0..7ad3ed8728f 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -4,7 +4,7 @@ module Gitlab
       key = :current_application_settings
 
       RequestStore.store[key] ||= begin
-        if ActiveRecord::Base.connected? && ActiveRecord::Base.connection.table_exists?('application_settings')
+        if ActiveRecord::Base.connection.active? && ActiveRecord::Base.connection.table_exists?('application_settings')
           ApplicationSetting.current || ApplicationSetting.create_from_defaults
         else
           fake_application_settings
-- 
cgit v1.2.3