Merge branch 'master' into joelkoglin/gitlab-ce-feature_fix_ldap_auth_issue_993

17 files changed, 429 insertions, 33 deletions

diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb
index 42c93707caa..d8a7d29f1bf 100644
--- a/lib/gitlab/bitbucket_import/importer.rb
+++ b/lib/gitlab/bitbucket_import/importer.rb
@@ -5,7 +5,10 @@ module Gitlab
 
       def initialize(project)
         @project = project
-        @client = Client.new(project.creator.bitbucket_access_token, project.creator.bitbucket_access_token_secret)
+        import_data = project.import_data.try(:data)
+        bb_session = import_data["bb_session"] if import_data
+        @client = Client.new(bb_session["bitbucket_access_token"],
+                             bb_session["bitbucket_access_token_secret"])
         @formatter = Gitlab::ImportFormatter.new
       end
 
@@ -16,12 +19,12 @@ module Gitlab
 
         #Issues && Comments
         issues = client.issues(project_identifier)
-        
+
         issues["issues"].each do |issue|
           body = @formatter.author_line(issue["reported_by"]["username"], issue["content"])
-          
+
           comments = client.issue_comments(project_identifier, issue["local_id"])
-          
+
           if comments.any?
             body += @formatter.comments_header
           end
@@ -31,13 +34,13 @@ module Gitlab
           end
 
           project.issues.create!(
-            description: body, 
+            description: body,
             title: issue["title"],
             state: %w(resolved invalid duplicate wontfix).include?(issue["status"]) ? 'closed' : 'opened',
             author_id: gl_user_id(project, issue["reported_by"]["username"])
           )
         end
-        
+
         true
       end
 
diff --git a/lib/gitlab/bitbucket_import/key_adder.rb b/lib/gitlab/bitbucket_import/key_adder.rb
index 9931aa7e029..0b63f025d0a 100644
--- a/lib/gitlab/bitbucket_import/key_adder.rb
+++ b/lib/gitlab/bitbucket_import/key_adder.rb
@@ -3,14 +3,15 @@ module Gitlab
     class KeyAdder
       attr_reader :repo, :current_user, :client
 
-      def initialize(repo, current_user)
+      def initialize(repo, current_user, access_params)
         @repo, @current_user = repo, current_user
-        @client = Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret)
+        @client = Client.new(access_params[:bitbucket_access_token],
+                             access_params[:bitbucket_access_token_secret])
       end
 
       def execute
         return false unless BitbucketImport.public_key.present?
-        
+
         project_identifier = "#{repo["owner"]}/#{repo["slug"]}"
         client.add_deploy_key(project_identifier, BitbucketImport.public_key)
 
diff --git a/lib/gitlab/bitbucket_import/key_deleter.rb b/lib/gitlab/bitbucket_import/key_deleter.rb
index 1a24a86fc37..f4dd393ad29 100644
--- a/lib/gitlab/bitbucket_import/key_deleter.rb
+++ b/lib/gitlab/bitbucket_import/key_deleter.rb
@@ -6,12 +6,15 @@ module Gitlab
       def initialize(project)
         @project = project
         @current_user = project.creator
-        @client = Client.new(current_user.bitbucket_access_token, current_user.bitbucket_access_token_secret)
+        import_data = project.import_data.try(:data)
+        bb_session = import_data["bb_session"] if import_data
+        @client = Client.new(bb_session["bitbucket_access_token"],
+                             bb_session["bitbucket_access_token_secret"])
       end
 
       def execute
         return false unless BitbucketImport.public_key.present?
-        
+
         client.delete_deploy_key(project.import_source, BitbucketImport.public_key)
 
         true
diff --git a/lib/gitlab/bitbucket_import/project_creator.rb b/lib/gitlab/bitbucket_import/project_creator.rb
index 54420e62c90..35e34d033e0 100644
--- a/lib/gitlab/bitbucket_import/project_creator.rb
+++ b/lib/gitlab/bitbucket_import/project_creator.rb
@@ -1,16 +1,17 @@
 module Gitlab
   module BitbucketImport
     class ProjectCreator
-      attr_reader :repo, :namespace, :current_user
+      attr_reader :repo, :namespace, :current_user, :session_data
 
-      def initialize(repo, namespace, current_user)
+      def initialize(repo, namespace, current_user, session_data)
         @repo = repo
         @namespace = namespace
         @current_user = current_user
+        @session_data = session_data
       end
 
       def execute
-        ::Projects::CreateService.new(current_user,
+        project = ::Projects::CreateService.new(current_user,
           name: repo["name"],
           path: repo["slug"],
           description: repo["description"],
@@ -18,8 +19,11 @@ module Gitlab
           visibility_level: repo["is_private"] ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
           import_type: "bitbucket",
           import_source: "#{repo["owner"]}/#{repo["slug"]}",
-          import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git"
+          import_url: "ssh://git@bitbucket.org/#{repo["owner"]}/#{repo["slug"]}.git",
         ).execute
+
+        project.create_import_data(data: { "bb_session" => session_data } )
+        project
       end
     end
   end
diff --git a/lib/gitlab/color_schemes.rb b/lib/gitlab/color_schemes.rb
new file mode 100644
index 00000000000..9c4664df903
--- /dev/null
+++ b/lib/gitlab/color_schemes.rb
@@ -0,0 +1,67 @@
+module Gitlab
+  # Module containing GitLab's syntax color scheme definitions and helper
+  # methods for accessing them.
+  module ColorSchemes
+    # Struct class representing a single Scheme
+    Scheme = Struct.new(:id, :name, :css_class)
+
+    SCHEMES = [
+      Scheme.new(1, 'White',           'white'),
+      Scheme.new(2, 'Dark',            'dark'),
+      Scheme.new(3, 'Solarized Light', 'solarized-light'),
+      Scheme.new(4, 'Solarized Dark',  'solarized-dark'),
+      Scheme.new(5, 'Monokai',         'monokai')
+    ].freeze
+
+    # Convenience method to get a space-separated String of all the color scheme
+    # classes that might be applied to a code block.
+    #
+    # Returns a String
+    def self.body_classes
+      SCHEMES.collect(&:css_class).uniq.join(' ')
+    end
+
+    # Get a Scheme by its ID
+    #
+    # If the ID is invalid, returns the default Scheme.
+    #
+    # id - Integer ID
+    #
+    # Returns a Scheme
+    def self.by_id(id)
+      SCHEMES.detect { |s| s.id == id } || default
+    end
+
+    # Returns the number of defined Schemes
+    def self.count
+      SCHEMES.size
+    end
+
+    # Get the default Scheme
+    #
+    # Returns a Scheme
+    def self.default
+      by_id(1)
+    end
+
+    # Iterate through each Scheme
+    #
+    # Yields the Scheme object
+    def self.each(&block)
+      SCHEMES.each(&block)
+    end
+
+    # Get the Scheme for the specified user, or the default
+    #
+    # user - User record
+    #
+    # Returns a Scheme
+    def self.for_user(user)
+      if user
+        by_id(user.color_scheme_id)
+      else
+        default
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 1a2a50a14d0..7ad3ed8728f 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -4,7 +4,7 @@ module Gitlab
       key = :current_application_settings
 
       RequestStore.store[key] ||= begin
-        if ActiveRecord::Base.connected? && ActiveRecord::Base.connection.table_exists?('application_settings')
+        if ActiveRecord::Base.connection.active? && ActiveRecord::Base.connection.table_exists?('application_settings')
           ApplicationSetting.current || ApplicationSetting.create_from_defaults
         else
           fake_application_settings
diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb
new file mode 100644
index 00000000000..32cece8316b
--- /dev/null
+++ b/lib/gitlab/email/attachment_uploader.rb
@@ -0,0 +1,35 @@
+module Gitlab
+  module Email
+    class AttachmentUploader
+      attr_accessor :message
+
+      def initialize(message)
+        @message = message
+      end
+
+      def execute(project)
+        attachments = []
+
+        message.attachments.each do |attachment|
+          tmp = Tempfile.new("gitlab-email-attachment")
+          begin
+            File.open(tmp.path, "w+b") { |f| f.write attachment.body.decoded }
+
+            file = {
+              tempfile:     tmp,
+              filename:     attachment.filename,
+              content_type: attachment.content_type
+            }
+
+            link = ::Projects::UploadService.new(project, file).execute
+            attachments << link if link
+          ensure
+            tmp.close!
+          end
+        end
+
+        attachments
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb
new file mode 100644
index 00000000000..355fbd27898
--- /dev/null
+++ b/lib/gitlab/email/receiver.rb
@@ -0,0 +1,106 @@
+# Inspired in great part by Discourse's Email::Receiver
+module Gitlab
+  module Email
+    class Receiver
+      class ProcessingError < StandardError; end
+      class EmailUnparsableError < ProcessingError; end
+      class SentNotificationNotFoundError < ProcessingError; end
+      class EmptyEmailError < ProcessingError; end
+      class AutoGeneratedEmailError < ProcessingError; end
+      class UserNotFoundError < ProcessingError; end
+      class UserBlockedError < ProcessingError; end
+      class UserNotAuthorizedError < ProcessingError; end
+      class NoteableNotFoundError < ProcessingError; end
+      class InvalidNoteError < ProcessingError; end
+
+      def initialize(raw)
+        @raw = raw
+      end
+
+      def execute
+        raise EmptyEmailError if @raw.blank?
+
+        raise SentNotificationNotFoundError unless sent_notification
+
+        raise AutoGeneratedEmailError if message.header.to_s =~ /auto-(generated|replied)/
+
+        author = sent_notification.recipient
+
+        raise UserNotFoundError unless author
+
+        raise UserBlockedError if author.blocked?
+
+        project = sent_notification.project
+
+        raise UserNotAuthorizedError unless project && author.can?(:create_note, project)
+
+        raise NoteableNotFoundError unless sent_notification.noteable
+
+        reply = ReplyParser.new(message).execute.strip
+
+        raise EmptyEmailError if reply.blank?
+
+        reply = add_attachments(reply)
+
+        note = create_note(reply)
+
+        unless note.persisted?
+          message = "The comment could not be created for the following reasons:"
+          note.errors.full_messages.each do |error|
+            message << "\n\n- #{error}"
+          end
+
+          raise InvalidNoteError, message
+        end
+      end
+
+      private
+
+      def message
+        @message ||= Mail::Message.new(@raw)
+      rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError => e
+        raise EmailUnparsableError, e
+      end
+
+      def reply_key
+        reply_key = nil
+        message.to.each do |address|
+          reply_key = Gitlab::ReplyByEmail.reply_key_from_address(address)
+          break if reply_key
+        end
+
+        reply_key
+      end
+
+      def sent_notification
+        return nil unless reply_key
+        
+        SentNotification.for(reply_key)
+      end
+
+      def add_attachments(reply)
+        attachments = Email::AttachmentUploader.new(message).execute(sent_notification.project)
+
+        attachments.each do |link|
+          text = "[#{link[:alt]}](#{link[:url]})"
+          text.prepend("!") if link[:is_image]
+
+          reply << "\n\n#{text}"
+        end
+
+        reply
+      end
+
+      def create_note(reply)
+        Notes::CreateService.new(
+          sent_notification.project,
+          sent_notification.recipient,
+          note:           reply,
+          noteable_type:  sent_notification.noteable_type,
+          noteable_id:    sent_notification.noteable_id,
+          commit_id:      sent_notification.commit_id
+        ).execute
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/email/reply_parser.rb b/lib/gitlab/email/reply_parser.rb
new file mode 100644
index 00000000000..6ed36b51f12
--- /dev/null
+++ b/lib/gitlab/email/reply_parser.rb
@@ -0,0 +1,79 @@
+# Inspired in great part by Discourse's Email::Receiver
+module Gitlab
+  module Email
+    class ReplyParser
+      attr_accessor :message
+
+      def initialize(message)
+        @message = message
+      end
+
+      def execute
+        body = select_body(message)
+
+        encoding = body.encoding
+
+        body = discourse_email_trimmer(body)
+
+        body = EmailReplyParser.parse_reply(body)
+
+        body.force_encoding(encoding).encode("UTF-8")
+      end
+
+      private
+
+      def select_body(message)
+        text    = message.text_part if message.multipart?
+        text  ||= message           if message.content_type !~ /text\/html/
+
+        return "" unless text
+
+        text = fix_charset(text)
+
+        # Certain trigger phrases that means we didn't parse correctly
+        if text =~ /(Content\-Type\:|multipart\/alternative|text\/plain)/
+          return ""
+        end
+
+        text
+      end
+
+      # Force encoding to UTF-8 on a Mail::Message or Mail::Part
+      def fix_charset(object)
+        return nil if object.nil?
+
+        if object.charset
+          object.body.decoded.force_encoding(object.charset.gsub(/utf8/i, "UTF-8")).encode("UTF-8").to_s
+        else
+          object.body.to_s
+        end
+      rescue
+        nil
+      end
+
+      REPLYING_HEADER_LABELS = %w(From Sent To Subject Reply To Cc Bcc Date)
+      REPLYING_HEADER_REGEX = Regexp.union(REPLYING_HEADER_LABELS.map { |label| "#{label}:" })
+
+      def discourse_email_trimmer(body)
+        lines = body.scrub.lines.to_a
+        range_end = 0
+
+        lines.each_with_index do |l, idx|
+          # This one might be controversial but so many reply lines have years, times and end with a colon.
+          # Let's try it and see how well it works.
+          break if (l =~ /\d{4}/ && l =~ /\d:\d\d/ && l =~ /\:$/) ||
+                   (l =~ /On \w+ \d+,? \d+,?.*wrote:/)
+
+          # Headers on subsequent lines
+          break if (0..2).all? { |off| lines[idx+off] =~ REPLYING_HEADER_REGEX }
+          # Headers on the same line
+          break if REPLYING_HEADER_LABELS.count { |label| l.include?(label) } >= 3
+
+          range_end = idx
+        end
+
+        lines[0..range_end].join.strip
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/github_import/importer.rb b/lib/gitlab/github_import/importer.rb
index 98039a76dcd..8c106a61735 100644
--- a/lib/gitlab/github_import/importer.rb
+++ b/lib/gitlab/github_import/importer.rb
@@ -5,7 +5,9 @@ module Gitlab
 
       def initialize(project)
         @project = project
-        @client = Client.new(project.creator.github_access_token)
+        import_data = project.import_data.try(:data)
+        github_session = import_data["github_session"] if import_data
+        @client = Client.new(github_session["github_access_token"])
         @formatter = Gitlab::ImportFormatter.new
       end
 
diff --git a/lib/gitlab/github_import/project_creator.rb b/lib/gitlab/github_import/project_creator.rb
index 2723eec933e..8c27ebd1ce8 100644
--- a/lib/gitlab/github_import/project_creator.rb
+++ b/lib/gitlab/github_import/project_creator.rb
@@ -1,16 +1,18 @@
 module Gitlab
   module GithubImport
     class ProjectCreator
-      attr_reader :repo, :namespace, :current_user
+      attr_reader :repo, :namespace, :current_user, :session_data
 
-      def initialize(repo, namespace, current_user)
+      def initialize(repo, namespace, current_user, session_data)
         @repo = repo
         @namespace = namespace
         @current_user = current_user
+        @session_data = session_data
       end
 
       def execute
-        ::Projects::CreateService.new(current_user,
+        project = ::Projects::CreateService.new(
+          current_user,
           name: repo.name,
           path: repo.name,
           description: repo.description,
@@ -18,8 +20,11 @@ module Gitlab
           visibility_level: repo.private ? Gitlab::VisibilityLevel::PRIVATE : Gitlab::VisibilityLevel::PUBLIC,
           import_type: "github",
           import_source: repo.full_name,
-          import_url: repo.clone_url.sub("https://", "https://#{current_user.github_access_token}@")
+          import_url: repo.clone_url.sub("https://", "https://#{@session_data[:github_access_token]}@")
         ).execute
+
+        project.create_import_data(data: { "github_session" => session_data } )
+        project
       end
     end
   end
diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb
index c5304a0699b..50594d2b24f 100644
--- a/lib/gitlab/gitlab_import/importer.rb
+++ b/lib/gitlab/gitlab_import/importer.rb
@@ -5,7 +5,9 @@ module Gitlab
 
       def initialize(project)
         @project = project
-        @client = Client.new(project.creator.gitlab_access_token)
+        import_data = project.import_data.try(:data)
+        gitlab_session = import_data["gitlab_session"] if import_data
+        @client = Client.new(gitlab_session["gitlab_access_token"])
         @formatter = Gitlab::ImportFormatter.new
       end
 
@@ -14,12 +16,12 @@ module Gitlab
 
         #Issues && Comments
         issues = client.issues(project_identifier)
-        
+
         issues.each do |issue|
           body = @formatter.author_line(issue["author"]["name"], issue["description"])
-          
+
           comments = client.issue_comments(project_identifier, issue["id"])
-          
+
           if comments.any?
             body += @formatter.comments_header
           end
@@ -29,13 +31,13 @@ module Gitlab
           end
 
           project.issues.create!(
-            description: body, 
+            description: body,
             title: issue["title"],
             state: issue["state"],
             author_id: gl_user_id(project, issue["author"]["id"])
           )
         end
-        
+
         true
       end
 
diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb
index f0d7141bf56..d9452de6a50 100644
--- a/lib/gitlab/gitlab_import/project_creator.rb
+++ b/lib/gitlab/gitlab_import/project_creator.rb
@@ -1,16 +1,17 @@
 module Gitlab
   module GitlabImport
     class ProjectCreator
-      attr_reader :repo, :namespace, :current_user
+      attr_reader :repo, :namespace, :current_user, :session_data
 
-      def initialize(repo, namespace, current_user)
+      def initialize(repo, namespace, current_user, session_data)
         @repo = repo
         @namespace = namespace
         @current_user = current_user
+        @session_data = session_data
       end
 
       def execute
-        ::Projects::CreateService.new(current_user,
+        project = ::Projects::CreateService.new(current_user,
           name: repo["name"],
           path: repo["path"],
           description: repo["description"],
@@ -18,8 +19,11 @@ module Gitlab
           visibility_level: repo["visibility_level"],
           import_type: "gitlab",
           import_source: repo["path_with_namespace"],
-          import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{current_user.gitlab_access_token}@")
+          import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@")
         ).execute
+
+        project.create_import_data(data: { "gitlab_session" => session_data } )
+        project
       end
     end
   end
diff --git a/lib/gitlab/markdown/autolink_filter.rb b/lib/gitlab/markdown/autolink_filter.rb
index 4e14a048cfb..541f1d88ffc 100644
--- a/lib/gitlab/markdown/autolink_filter.rb
+++ b/lib/gitlab/markdown/autolink_filter.rb
@@ -87,8 +87,14 @@ module Gitlab
 
       def autolink_filter(text)
         text.gsub(LINK_PATTERN) do |match|
+          # Remove any trailing HTML entities and store them for appending
+          # outside the link element. The entity must be marked HTML safe in
+          # order to be output literally rather than escaped.
+          match.gsub!(/((?:&[\w#]+;)+)\z/, '')
+          dropped = ($1 || '').html_safe
+
           options = link_options.merge(href: match)
-          content_tag(:a, match, options)
+          content_tag(:a, match, options) + dropped
         end
       end
 
diff --git a/lib/gitlab/reply_by_email.rb b/lib/gitlab/reply_by_email.rb
new file mode 100644
index 00000000000..c3fe6778f06
--- /dev/null
+++ b/lib/gitlab/reply_by_email.rb
@@ -0,0 +1,49 @@
+module Gitlab
+  module ReplyByEmail
+    class << self
+      def enabled?
+        config.enabled && address_formatted_correctly?
+      end
+
+      def address_formatted_correctly?
+        config.address &&
+          config.address.include?("%{reply_key}")
+      end
+
+      def reply_key
+        return nil unless enabled?
+
+        SecureRandom.hex(16)
+      end
+
+      def reply_address(reply_key)
+        config.address.gsub('%{reply_key}', reply_key)
+      end
+
+      def reply_key_from_address(address)
+        regex = address_regex
+        return unless regex
+
+        match = address.match(regex)
+        return unless match
+
+        match[1]
+      end
+
+      private
+
+      def config
+        Gitlab.config.reply_by_email
+      end
+
+      def address_regex
+        wildcard_address = config.address
+        return nil unless wildcard_address
+
+        regex = Regexp.escape(wildcard_address)
+        regex = regex.gsub(Regexp.escape('%{reply_key}'), "(.+)")
+        Regexp.new(regex).freeze
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index 06245374bc8..2ab2d4af797 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -19,13 +19,15 @@ module Gitlab
         issues.page(page).per(per_page)
       when 'merge_requests'
         merge_requests.page(page).per(per_page)
+      when 'milestones'
+        milestones.page(page).per(per_page)
       else
         Kaminari.paginate_array([]).page(page).per(per_page)
       end
     end
 
     def total_count
-      @total_count ||= projects_count + issues_count + merge_requests_count
+      @total_count ||= projects_count + issues_count + merge_requests_count + milestones_count
     end
 
     def projects_count
@@ -40,6 +42,10 @@ module Gitlab
       @merge_requests_count ||= merge_requests.count
     end
 
+    def milestones_count
+      @milestones_count ||= milestones.count
+    end
+
     def empty?
       total_count.zero?
     end
@@ -60,6 +66,12 @@ module Gitlab
       issues.order('updated_at DESC')
     end
 
+    def milestones
+      milestones = Milestone.where(project_id: limit_project_ids)
+      milestones = milestones.search(query)
+      milestones.order('updated_at DESC')
+    end
+
     def merge_requests
       merge_requests = MergeRequest.in_projects(limit_project_ids)
       if query =~ /[#!](\d+)\z/
diff --git a/lib/gitlab/themes.rb b/lib/gitlab/themes.rb
index 5209df92795..83f91de810c 100644
--- a/lib/gitlab/themes.rb
+++ b/lib/gitlab/themes.rb
@@ -37,6 +37,11 @@ module Gitlab
       THEMES.detect { |t| t.id == id } || default
     end
 
+    # Returns the number of defined Themes
+    def self.count
+      THEMES.size
+    end
+
     # Get the default Theme
     #
     # Returns a Theme
@@ -51,6 +56,19 @@ module Gitlab
       THEMES.each(&block)
     end
 
+    # Get the Theme for the specified user, or the default
+    #
+    # user - User record
+    #
+    # Returns a Theme
+    def self.for_user(user)
+      if user
+        by_id(user.theme_id)
+      else
+        default
+      end
+    end
+
     private
 
     def self.default_id