Merge remote-tracking branch 'origin/master' into feature-oauth-refactoring

12 files changed, 334 insertions, 93 deletions

diff --git a/lib/gitlab/backend/shell.rb b/lib/gitlab/backend/shell.rb
index 53bff3037e5..907373ab991 100644
--- a/lib/gitlab/backend/shell.rb
+++ b/lib/gitlab/backend/shell.rb
@@ -107,12 +107,17 @@ module Gitlab
     # path - project path with namespace
     # tag_name - new tag name
     # ref - HEAD for new tag
+    # message - optional message for tag (annotated tag)
     #
     # Ex.
     #   add_tag("gitlab/gitlab-ci", "v4.0", "master")
+    #   add_tag("gitlab/gitlab-ci", "v4.0", "master", "message")
     #
-    def add_tag(path, tag_name, ref)
-      system "#{gitlab_shell_path}/bin/gitlab-projects", "create-tag", "#{path}.git", tag_name, ref
+    def add_tag(path, tag_name, ref, message = nil)
+      cmd = %W(#{gitlab_shell_path}/bin/gitlab-projects create-tag #{path}.git
+               #{tag_name} #{ref})
+      cmd << message unless message.nil? || message.empty?
+      system *cmd
     end
 
     # Remove repository tag
diff --git a/lib/gitlab/blacklist.rb b/lib/gitlab/blacklist.rb
index 65efb6e4407..43145e0ee1b 100644
--- a/lib/gitlab/blacklist.rb
+++ b/lib/gitlab/blacklist.rb
@@ -27,6 +27,7 @@ module Gitlab
         notes
         unsubscribes
         all
+        ci
       )
     end
   end
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
new file mode 100644
index 00000000000..19a1198c68c
--- /dev/null
+++ b/lib/gitlab/diff/file.rb
@@ -0,0 +1,49 @@
+module Gitlab
+  module Diff
+    class File
+      attr_reader :diff
+
+      delegate :new_file, :deleted_file, :renamed_file,
+        :old_path, :new_path, to: :diff, prefix: false
+
+      def initialize(diff)
+        @diff = diff
+      end
+
+      # Array of Gitlab::DIff::Line objects
+      def diff_lines
+        @lines ||= parser.parse(raw_diff.lines)
+      end
+
+      def mode_changed?
+        !!(diff.a_mode && diff.b_mode && diff.a_mode != diff.b_mode)
+      end
+
+      def parser
+        Gitlab::Diff::Parser.new
+      end
+
+      def raw_diff
+        diff.diff
+      end
+
+      def next_line(index)
+        diff_lines[index + 1]
+      end
+
+      def prev_line(index)
+        if index > 0
+          diff_lines[index - 1]
+        end
+      end
+
+      def file_path
+        if diff.new_path.present?
+          diff.new_path
+        elsif diff.old_path.present?
+          diff.old_path
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
new file mode 100644
index 00000000000..8ac1b15e88a
--- /dev/null
+++ b/lib/gitlab/diff/line.rb
@@ -0,0 +1,12 @@
+module Gitlab
+  module Diff
+    class Line
+      attr_reader :type, :text, :index, :old_pos, :new_pos
+
+      def initialize(text, type, index, old_pos, new_pos)
+        @text, @type, @index = text, type, index
+        @old_pos, @new_pos = old_pos, new_pos
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/line_code.rb b/lib/gitlab/diff/line_code.rb
new file mode 100644
index 00000000000..f3578ab3d35
--- /dev/null
+++ b/lib/gitlab/diff/line_code.rb
@@ -0,0 +1,9 @@
+module Gitlab
+  module Diff
+    class LineCode
+      def self.generate(file_path, new_line_position, old_line_position)
+        "#{Digest::SHA1.hexdigest(file_path)}_#{old_line_position}_#{new_line_position}"
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb
new file mode 100644
index 00000000000..9d6309954a4
--- /dev/null
+++ b/lib/gitlab/diff/parser.rb
@@ -0,0 +1,81 @@
+module Gitlab
+  module Diff
+    class Parser
+      include Enumerable
+
+      def parse(lines)
+        @lines = lines,
+        lines_obj = []
+        line_obj_index = 0
+        line_old = 1
+        line_new = 1
+        type = nil
+
+        lines_arr = ::Gitlab::InlineDiff.processing lines
+
+        lines_arr.each do |line|
+          raw_line = line.dup
+
+          next if filename?(line)
+
+          full_line = html_escape(line.gsub(/\n/, ''))
+          full_line = ::Gitlab::InlineDiff.replace_markers full_line
+
+          if line.match(/^@@ -/)
+            type = "match"
+
+            line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0
+            line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0
+
+            next if line_old == 1 && line_new == 1 #top of file
+            lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new)
+            line_obj_index += 1
+            next
+          else
+            type = identification_type(line)
+            lines_obj << Gitlab::Diff::Line.new(full_line, type, line_obj_index, line_old, line_new)
+            line_obj_index += 1
+          end
+
+
+          if line[0] == "+"
+            line_new += 1
+          elsif line[0] == "-"
+            line_old += 1
+          else
+            line_new += 1
+            line_old += 1
+          end
+        end
+
+        lines_obj
+      end
+
+      def empty?
+        @lines.empty?
+      end
+
+      private
+
+      def filename?(line)
+        line.start_with?('--- /dev/null', '+++ /dev/null', '--- a', '+++ b',
+                         '--- /tmp/diffy', '+++ /tmp/diffy')
+      end
+
+      def identification_type(line)
+        if line[0] == "+"
+          "new"
+        elsif line[0] == "-"
+          "old"
+        else
+          nil
+        end
+      end
+
+      def html_escape str
+        replacements = { '&' => '&amp;', '>' => '&gt;', '<' => '&lt;', '"' => '&quot;', "'" => '&#39;' }
+          str.gsub(/[&"'><]/, replacements)
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/diff_parser.rb b/lib/gitlab/diff_parser.rb
deleted file mode 100644
index b244295027e..00000000000
--- a/lib/gitlab/diff_parser.rb
+++ /dev/null
@@ -1,83 +0,0 @@
-module Gitlab
-  class DiffParser
-    include Enumerable
-
-    attr_reader :lines, :new_path
-
-    def initialize(lines, new_path = '')
-      @lines = lines
-      @new_path = new_path
-    end
-
-    def each
-      line_old = 1
-      line_new = 1
-      type = nil
-
-      lines_arr = ::Gitlab::InlineDiff.processing lines
-      lines_arr.each do |line|
-        raw_line = line.dup
-
-        next if filename?(line)
-
-        full_line = html_escape(line.gsub(/\n/, ''))
-        full_line = ::Gitlab::InlineDiff.replace_markers full_line
-
-        if line.match(/^@@ -/)
-          type = "match"
-
-          line_old = line.match(/\-[0-9]*/)[0].to_i.abs rescue 0
-          line_new = line.match(/\+[0-9]*/)[0].to_i.abs rescue 0
-
-          next if line_old == 1 && line_new == 1 #top of file
-          yield(full_line, type, nil, line_new, line_old)
-          next
-        else
-          type = identification_type(line)
-          line_code = generate_line_code(new_path, line_new, line_old)
-          yield(full_line, type, line_code, line_new, line_old, raw_line)
-        end
-
-
-        if line[0] == "+"
-          line_new += 1
-        elsif line[0] == "-"
-          line_old += 1
-        else
-          line_new += 1
-          line_old += 1
-        end
-      end
-    end
-
-    def empty?
-      @lines.empty?
-    end
-
-    private
-
-    def filename?(line)
-      line.start_with?('--- /dev/null', '+++ /dev/null', '--- a', '+++ b',
-                       '--- /tmp/diffy', '+++ /tmp/diffy')
-    end
-
-    def identification_type(line)
-      if line[0] == "+"
-        "new"
-      elsif line[0] == "-"
-        "old"
-      else
-        nil
-      end
-    end
-
-    def generate_line_code(path, line_new, line_old)
-      "#{Digest::SHA1.hexdigest(path)}_#{line_old}_#{line_new}"
-    end
-
-    def html_escape str
-      replacements = { '&' => '&amp;', '>' => '&gt;', '<' => '&lt;', '"' => '&quot;', "'" => '&#39;' }
-        str.gsub(/[&"'><]/, replacements)
-    end
-  end
-end
diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb
new file mode 100644
index 00000000000..13cb08948bb
--- /dev/null
+++ b/lib/gitlab/git_ref_validator.rb
@@ -0,0 +1,11 @@
+module Gitlab
+  module GitRefValidator
+    extend self
+    # Validates a given name against the git reference specification
+    #
+    # Returns true for a valid reference name, false otherwise
+    def validate(ref_name)
+      system *%W(git check-ref-format refs/#{ref_name})
+    end
+  end
+end
diff --git a/lib/gitlab/markdown.rb b/lib/gitlab/markdown.rb
index b248d8f9436..6017a4c86c1 100644
--- a/lib/gitlab/markdown.rb
+++ b/lib/gitlab/markdown.rb
@@ -192,8 +192,12 @@ module Gitlab
 
           link_to("##{identifier}", url, options)
         end
-      elsif project.issues_tracker == 'jira'
-        reference_jira_issue(identifier, project)
+      else
+        config = Gitlab.config
+        external_issue_tracker = config.issues_tracker[project.issues_tracker]
+        if external_issue_tracker.present?
+          reference_external_issue(identifier, external_issue_tracker, project)
+        end
       end
     end
 
@@ -229,15 +233,15 @@ module Gitlab
       end
     end
 
-    def reference_jira_issue(identifier, project = @project)
-      url = url_for_issue(identifier)
-      title = Gitlab.config.issues_tracker[project.issues_tracker]["title"]
+    def reference_external_issue(identifier, issue_tracker, project = @project)
+      url = url_for_issue(identifier, project)
+      title = issue_tracker['title']
 
       options = html_options.merge(
         title: "Issue in #{title}",
         class: "gfm gfm-issue #{html_options[:class]}"
       )
-      link_to("#{identifier}", url, options)
+      link_to("##{identifier}", url, options)
     end
   end
 end
diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb
index 90511662b20..9dc8b34d9c7 100644
--- a/lib/gitlab/project_search_results.rb
+++ b/lib/gitlab/project_search_results.rb
@@ -14,13 +14,16 @@ module Gitlab
         notes.page(page).per(per_page)
       when 'blobs'
         Kaminari.paginate_array(blobs).page(page).per(per_page)
+      when 'wiki_blobs'
+        Kaminari.paginate_array(wiki_blobs).page(page).per(per_page)
       else
         super
       end
     end
 
     def total_count
-      @total_count ||= issues_count + merge_requests_count + blobs_count + notes_count
+      @total_count ||= issues_count + merge_requests_count + blobs_count +
+                       notes_count + wiki_blobs_count
     end
 
     def blobs_count
@@ -31,6 +34,10 @@ module Gitlab
       @notes_count ||= notes.count
     end
 
+    def wiki_blobs_count
+      @wiki_blobs_count ||= wiki_blobs.count
+    end
+
     private
 
     def blobs
@@ -41,6 +48,20 @@ module Gitlab
       end
     end
 
+    def wiki_blobs
+      if project.wiki_enabled?
+        wiki_repo = Repository.new(ProjectWiki.new(project).path_with_namespace)
+
+        if wiki_repo.exists?
+          wiki_repo.search_files(query)
+        else
+          []
+        end
+      else
+        []
+      end
+    end
+
     def notes
       Note.where(project_id: limit_project_ids).search(query).order('updated_at DESC')
     end
diff --git a/lib/gitlab/snippet_search_results.rb b/lib/gitlab/snippet_search_results.rb
new file mode 100644
index 00000000000..938219efdb2
--- /dev/null
+++ b/lib/gitlab/snippet_search_results.rb
@@ -0,0 +1,131 @@
+module Gitlab
+  class SnippetSearchResults < SearchResults
+    attr_reader :limit_snippet_ids
+
+    def initialize(limit_snippet_ids, query)
+      @limit_snippet_ids = limit_snippet_ids
+      @query = query
+    end
+
+    def objects(scope, page = nil)
+      case scope
+      when 'snippet_titles'
+        Kaminari.paginate_array(snippet_titles).page(page).per(per_page)
+      when 'snippet_blobs'
+        Kaminari.paginate_array(snippet_blobs).page(page).per(per_page)
+      else
+        super
+      end
+    end
+
+    def total_count
+      @total_count ||= snippet_titles_count + snippet_blobs_count
+    end
+
+    def snippet_titles_count
+      @snippet_titles_count ||= snippet_titles.count
+    end
+
+    def snippet_blobs_count
+      @snippet_blobs_count ||= snippet_blobs.count
+    end
+
+    private
+
+    def snippet_titles
+      Snippet.where(id: limit_snippet_ids).search(query).order('updated_at DESC')
+    end
+
+    def snippet_blobs
+      search = Snippet.where(id: limit_snippet_ids).search_code(query)
+      search = search.order('updated_at DESC').to_a
+      snippets = []
+      search.each { |e| snippets << chunk_snippet(e) }
+      snippets
+    end
+
+    def default_scope
+      'snippet_blobs'
+    end
+
+    # Get an array of line numbers surrounding a matching
+    # line, bounded by min/max.
+    #
+    # @returns Array of line numbers
+    def bounded_line_numbers(line, min, max)
+      lower = line - surrounding_lines > min ? line - surrounding_lines : min
+      upper = line + surrounding_lines < max ? line + surrounding_lines : max
+      (lower..upper).to_a
+    end
+
+    # Returns a sorted set of lines to be included in a snippet preview.
+    # This ensures matching adjacent lines do not display duplicated
+    # surrounding code.
+    #
+    # @returns Array, unique and sorted.
+    def matching_lines(lined_content)
+      used_lines = []
+      lined_content.each_with_index do |line, line_number|
+        used_lines.concat bounded_line_numbers(
+          line_number,
+          0,
+          lined_content.size
+        ) if line.include?(query)
+      end
+
+      used_lines.uniq.sort
+    end
+
+    # 'Chunkify' entire snippet.  Splits the snippet data into matching lines +
+    # surrounding_lines() worth of unmatching lines.
+    #
+    # @returns a hash with {snippet_object, snippet_chunks:{data,start_line}}
+    def chunk_snippet(snippet)
+      lined_content = snippet.content.split("\n")
+      used_lines = matching_lines(lined_content)
+
+      snippet_chunk = []
+      snippet_chunks = []
+      snippet_start_line = 0
+      last_line = -1
+
+      # Go through each used line, and add consecutive lines as a single chunk
+      # to the snippet chunk array.
+      used_lines.each do |line_number|
+        if last_line < 0
+          # Start a new chunk.
+          snippet_start_line = line_number
+          snippet_chunk << lined_content[line_number]
+        elsif last_line == line_number - 1
+          # Consecutive line, continue chunk.
+          snippet_chunk << lined_content[line_number]
+        else
+          # Non-consecutive line, add chunk to chunk array.
+          snippet_chunks << {
+            data: snippet_chunk.join("\n"),
+            start_line: snippet_start_line + 1
+          }
+
+          # Start a new chunk.
+          snippet_chunk = [lined_content[line_number]]
+          snippet_start_line = line_number
+        end
+        last_line = line_number
+      end
+      # Add final chunk to chunk array
+      snippet_chunks << {
+        data: snippet_chunk.join("\n"),
+        start_line: snippet_start_line + 1
+      }
+
+      # Return snippet with chunk array
+      { snippet_object: snippet, snippet_chunks: snippet_chunks }
+    end
+
+    # Defines how many unmatching lines should be
+    # included around the matching lines in a snippet
+    def surrounding_lines
+      3
+    end
+  end
+end
diff --git a/lib/gitlab/upgrader.rb b/lib/gitlab/upgrader.rb
index 0846359f9b1..74b049b5143 100644
--- a/lib/gitlab/upgrader.rb
+++ b/lib/gitlab/upgrader.rb
@@ -43,7 +43,7 @@ module Gitlab
     end
 
     def latest_version_raw
-      remote_tags, _ = Gitlab::Popen.popen(%W(git ls-remote --tags origin))
+      remote_tags, _ = Gitlab::Popen.popen(%W(git ls-remote --tags https://gitlab.com/gitlab-org/gitlab-ce.git))
       git_tags = remote_tags.split("\n").grep(/tags\/v#{current_version.major}/)
       git_tags = git_tags.select { |version| version =~ /v\d\.\d\.\d\Z/ }
       last_tag = git_tags.last.match(/v\d\.\d\.\d/).to_s