Merge remote-tracking branch 'origin/master' into zj-job-view-goes-real-time

15 files changed, 134 insertions, 113 deletions

diff --git a/lib/gitlab/current_settings.rb b/lib/gitlab/current_settings.rb
index 82576d197fe..9e14b35b0f8 100644
--- a/lib/gitlab/current_settings.rb
+++ b/lib/gitlab/current_settings.rb
@@ -19,7 +19,7 @@ module Gitlab
           settings = ::ApplicationSetting.last
         end
 
-        settings ||= ::ApplicationSetting.create_from_defaults unless ActiveRecord::Migrator.needs_migration?
+        settings ||= ::ApplicationSetting.create_from_defaults
       end
 
       settings || in_memory_application_settings
@@ -46,7 +46,8 @@ module Gitlab
       active_db_connection = ActiveRecord::Base.connection.active? rescue false
 
       active_db_connection &&
-        ActiveRecord::Base.connection.table_exists?('application_settings')
+        ActiveRecord::Base.connection.table_exists?('application_settings') &&
+        !ActiveRecord::Migrator.needs_migration?
     rescue ActiveRecord::NoDatabaseError
       false
     end
diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb
index 79836a2fbab..a6007ebf531 100644
--- a/lib/gitlab/diff/file_collection/base.rb
+++ b/lib/gitlab/diff/file_collection/base.rb
@@ -7,7 +7,7 @@ module Gitlab
         delegate :count, :size, :real_size, to: :diff_files
 
         def self.default_options
-          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, no_collapse: false)
+          ::Commit.max_diff_options.merge(ignore_whitespace_change: false, expanded: false)
         end
 
         def initialize(diffable, project:, diff_options: nil, diff_refs: nil, fallback_diff_refs: nil)
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 0a15c6d9358..bd52ae47e9f 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -59,6 +59,10 @@ module Gitlab
         type == 'match'
       end
 
+      def discussable?
+        !['match', 'new-nonewline', 'old-nonewline'].include?(type)
+      end
+
       def as_json(opts = nil)
         {
           type: type,
diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb
index 6c69cd9e6a9..ea035e33eff 100644
--- a/lib/gitlab/email/message/repository_push.rb
+++ b/lib/gitlab/email/message/repository_push.rb
@@ -42,7 +42,7 @@ module Gitlab
           return unless compare
 
           # This diff is more moderated in number of files and lines
-          @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, no_collapse: true).diff_files
+          @diffs ||= compare.diffs(max_files: 30, max_lines: 5000, expanded: true).diff_files
         end
 
         def diffs_count
diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb
new file mode 100644
index 00000000000..dbe28e6bb93
--- /dev/null
+++ b/lib/gitlab/encoding_helper.rb
@@ -0,0 +1,62 @@
+module Gitlab
+  module EncodingHelper
+    extend self
+
+    # This threshold is carefully tweaked to prevent usage of encodings detected
+    # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
+    # we're better off sticking with utf8 encoding.
+    # Reason: git diff can return strings with invalid utf8 byte sequences if it
+    # truncates a diff in the middle of a multibyte character. In this case
+    # CharlockHolmes will try to guess the encoding and will likely suggest an
+    # obscure encoding with low confidence.
+    # There is a lot more info with this merge request:
+    # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
+    ENCODING_CONFIDENCE_THRESHOLD = 40
+
+    def encode!(message)
+      return nil unless message.respond_to? :force_encoding
+
+      # if message is utf-8 encoding, just return it
+      message.force_encoding("UTF-8")
+      return message if message.valid_encoding?
+
+      # return message if message type is binary
+      detect = CharlockHolmes::EncodingDetector.detect(message)
+      return message.force_encoding("BINARY") if detect && detect[:type] == :binary
+
+      # force detected encoding if we have sufficient confidence.
+      if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
+        message.force_encoding(detect[:encoding])
+      end
+
+      # encode and clean the bad chars
+      message.replace clean(message)
+    rescue
+      encoding = detect ? detect[:encoding] : "unknown"
+      "--broken encoding: #{encoding}"
+    end
+
+    def encode_utf8(message)
+      detect = CharlockHolmes::EncodingDetector.detect(message)
+      if detect
+        begin
+          CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
+        rescue ArgumentError => e
+          Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}")
+
+          ''
+        end
+      else
+        clean(message)
+      end
+    end
+
+    private
+
+    def clean(message)
+      message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
+        .encode("UTF-8")
+        .gsub("\0".encode("UTF-8"), "")
+    end
+  end
+end
diff --git a/lib/gitlab/etag_caching/router.rb b/lib/gitlab/etag_caching/router.rb
index 53f3f442bc3..ca49eda51fb 100644
--- a/lib/gitlab/etag_caching/router.rb
+++ b/lib/gitlab/etag_caching/router.rb
@@ -10,10 +10,10 @@ module Gitlab
       #   - Ending in `issues/id`/realtime_changes` for the `issue_title` route
       USED_IN_ROUTES = %w[noteable issue notes issues realtime_changes
                           commit pipelines merge_requests builds
-                          new].freeze
-
+                          new environments].freeze
       RESERVED_WORDS = Gitlab::PathRegex::ILLEGAL_PROJECT_PATH_WORDS - USED_IN_ROUTES
       RESERVED_WORDS_REGEX = Regexp.union(*RESERVED_WORDS.map(&Regexp.method(:escape)))
+
       ROUTES = [
         Gitlab::EtagCaching::Router::Route.new(
           %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/noteable/issue/\d+/notes\z),
@@ -46,6 +46,10 @@ module Gitlab
         Gitlab::EtagCaching::Router::Route.new(
           %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/builds/\d+\.json\z),
           'project_build'
+        ),
+        Gitlab::EtagCaching::Router::Route.new(
+          %r(^(?!.*(#{RESERVED_WORDS_REGEX})).*/environments\.json\z),
+          'environments'
         )
       ].freeze
 
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 58193391926..66829a03c2e 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Git
     class Blame
-      include Gitlab::Git::EncodingHelper
+      include Gitlab::EncodingHelper
 
       attr_reader :lines, :blames
 
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index c1b31618e0d..d60e607b02b 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -2,7 +2,7 @@ module Gitlab
   module Git
     class Blob
       include Linguist::BlobHelper
-      include Gitlab::Git::EncodingHelper
+      include Gitlab::EncodingHelper
 
       # This number is the maximum amount of data that we want to display to
       # the user. We load as much as we can for encoding detection
@@ -88,6 +88,7 @@ module Gitlab
           new(
             id: blob_entry[:oid],
             name: blob_entry[:name],
+            size: 0,
             data: '',
             path: path,
             commit_id: sha
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 297531db4cc..bb04731f08c 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -2,7 +2,7 @@
 module Gitlab
   module Git
     class Commit
-      include Gitlab::Git::EncodingHelper
+      include Gitlab::EncodingHelper
 
       attr_accessor :raw_commit, :head, :refs
 
diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb
index deade337354..0594ac8e213 100644
--- a/lib/gitlab/git/diff.rb
+++ b/lib/gitlab/git/diff.rb
@@ -3,7 +3,7 @@ module Gitlab
   module Git
     class Diff
       TimeoutError = Class.new(StandardError)
-      include Gitlab::Git::EncodingHelper
+      include Gitlab::EncodingHelper
 
       # Diff properties
       attr_accessor :old_path, :new_path, :a_mode, :b_mode, :diff
@@ -15,13 +15,16 @@ module Gitlab
       alias_method :deleted_file?, :deleted_file
       alias_method :renamed_file?, :renamed_file
 
+      attr_accessor :expanded
+
+      # We need this accessor because of `to_hash` and `init_from_hash`
       attr_accessor :too_large
 
       # The maximum size of a diff to display.
-      DIFF_SIZE_LIMIT = 102400 # 100 KB
+      SIZE_LIMIT = 100.kilobytes
 
       # The maximum size before a diff is collapsed.
-      DIFF_COLLAPSE_LIMIT = 10240 # 10 KB
+      COLLAPSE_LIMIT = 10.kilobytes
 
       class << self
         def between(repo, head, base, options = {}, *paths)
@@ -152,7 +155,7 @@ module Gitlab
                              :include_untracked_content, :skip_binary_check,
                              :include_typechange, :include_typechange_trees,
                              :ignore_filemode, :recurse_ignored_dirs, :paths,
-                             :max_files, :max_lines, :all_diffs, :no_collapse]
+                             :max_files, :max_lines, :limits, :expanded]
 
           if default_options
             actual_defaults = default_options.dup
@@ -177,16 +180,18 @@ module Gitlab
         end
       end
 
-      def initialize(raw_diff, collapse: false)
+      def initialize(raw_diff, expanded: true)
+        @expanded = expanded
+
         case raw_diff
         when Hash
           init_from_hash(raw_diff)
-          prune_diff_if_eligible(collapse)
+          prune_diff_if_eligible
         when Rugged::Patch, Rugged::Diff::Delta
-          init_from_rugged(raw_diff, collapse: collapse)
+          init_from_rugged(raw_diff)
         when Gitaly::CommitDiffResponse
           init_from_gitaly(raw_diff)
-          prune_diff_if_eligible(collapse)
+          prune_diff_if_eligible
         when Gitaly::CommitDelta
           init_from_gitaly(raw_diff)
         when nil
@@ -226,17 +231,13 @@ module Gitlab
 
       def too_large?
         if @too_large.nil?
-          @too_large = @diff.bytesize >= DIFF_SIZE_LIMIT
+          @too_large = @diff.bytesize >= SIZE_LIMIT
         else
           @too_large
         end
       end
 
-      def collapsible?
-        @diff.bytesize >= DIFF_COLLAPSE_LIMIT
-      end
-
-      def prune_large_diff!
+      def too_large!
         @diff = ''
         @line_count = 0
         @too_large = true
@@ -244,10 +245,11 @@ module Gitlab
 
       def collapsed?
         return @collapsed if defined?(@collapsed)
-        false
+
+        @collapsed = !expanded && @diff.bytesize >= COLLAPSE_LIMIT
       end
 
-      def prune_collapsed_diff!
+      def collapse!
         @diff = ''
         @line_count = 0
         @collapsed = true
@@ -255,9 +257,9 @@ module Gitlab
 
       private
 
-      def init_from_rugged(rugged, collapse: false)
+      def init_from_rugged(rugged)
         if rugged.is_a?(Rugged::Patch)
-          init_from_rugged_patch(rugged, collapse: collapse)
+          init_from_rugged_patch(rugged)
           d = rugged.delta
         else
           d = rugged
@@ -272,10 +274,10 @@ module Gitlab
         @deleted_file = d.deleted?
       end
 
-      def init_from_rugged_patch(patch, collapse: false)
+      def init_from_rugged_patch(patch)
         # Don't bother initializing diffs that are too large. If a diff is
         # binary we're not going to display anything so we skip the size check.
-        return if !patch.delta.binary? && prune_large_patch(patch, collapse)
+        return if !patch.delta.binary? && prune_large_patch(patch)
 
         @diff = encode!(strip_diff_headers(patch.to_s))
       end
@@ -299,29 +301,32 @@ module Gitlab
         @deleted_file = msg.to_id == BLANK_SHA
       end
 
-      def prune_diff_if_eligible(collapse = false)
-        prune_large_diff! if too_large?
-        prune_collapsed_diff! if collapse && collapsible?
+      def prune_diff_if_eligible
+        if too_large?
+          too_large!
+        elsif collapsed?
+          collapse!
+        end
       end
 
       # If the patch surpasses any of the diff limits it calls the appropiate
       # prune method and returns true. Otherwise returns false.
-      def prune_large_patch(patch, collapse)
+      def prune_large_patch(patch)
         size = 0
 
         patch.each_hunk do |hunk|
           hunk.each_line do |line|
             size += line.content.bytesize
 
-            if size >= DIFF_SIZE_LIMIT
-              prune_large_diff!
+            if size >= SIZE_LIMIT
+              too_large!
               return true
             end
           end
         end
 
-        if collapse && size >= DIFF_COLLAPSE_LIMIT
-          prune_collapsed_diff!
+        if !expanded && size >= COLLAPSE_LIMIT
+          collapse!
           return true
         end
 
diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb
index 898a5ae15f2..334e06a6eca 100644
--- a/lib/gitlab/git/diff_collection.rb
+++ b/lib/gitlab/git/diff_collection.rb
@@ -9,12 +9,12 @@ module Gitlab
         @iterator = iterator
         @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files])
         @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines])
-        @max_bytes = @max_files * 5120 # Average 5 KB per file
+        @max_bytes = @max_files * 5.kilobytes # Average 5 KB per file
         @safe_max_files = [@max_files, DEFAULT_LIMITS[:max_files]].min
         @safe_max_lines = [@max_lines, DEFAULT_LIMITS[:max_lines]].min
-        @safe_max_bytes = @safe_max_files * 5120 # Average 5 KB per file
-        @all_diffs = !!options.fetch(:all_diffs, false)
-        @no_collapse = !!options.fetch(:no_collapse, true)
+        @safe_max_bytes = @safe_max_files * 5.kilobytes # Average 5 KB per file
+        @enforce_limits = !!options.fetch(:limits, true)
+        @expanded = !!options.fetch(:expanded, true)
 
         @line_count = 0
         @byte_count = 0
@@ -88,23 +88,23 @@ module Gitlab
         @iterator.each do |raw|
           @empty = false
 
-          if !@all_diffs && i >= @max_files
+          if @enforce_limits && i >= @max_files
             @overflow = true
             break
           end
 
-          collapse = !@all_diffs && !@no_collapse
+          expanded = !@enforce_limits || @expanded
 
-          diff = Gitlab::Git::Diff.new(raw, collapse: collapse)
+          diff = Gitlab::Git::Diff.new(raw, expanded: expanded)
 
-          if collapse && over_safe_limits?(i)
-            diff.prune_collapsed_diff!
+          if !expanded && over_safe_limits?(i)
+            diff.collapse!
           end
 
           @line_count += diff.line_count
           @byte_count += diff.diff.bytesize
 
-          if !@all_diffs && (@line_count >= @max_lines || @byte_count >= @max_bytes)
+          if @enforce_limits && (@line_count >= @max_lines || @byte_count >= @max_bytes)
             # This last Diff instance pushes us over the lines limit. We stop and
             # discard it.
             @overflow = true
diff --git a/lib/gitlab/git/encoding_helper.rb b/lib/gitlab/git/encoding_helper.rb
deleted file mode 100644
index f918074cb14..00000000000
--- a/lib/gitlab/git/encoding_helper.rb
+++ /dev/null
@@ -1,64 +0,0 @@
-module Gitlab
-  module Git
-    module EncodingHelper
-      extend self
-
-      # This threshold is carefully tweaked to prevent usage of encodings detected
-      # by CharlockHolmes with low confidence. If CharlockHolmes confidence is low,
-      # we're better off sticking with utf8 encoding.
-      # Reason: git diff can return strings with invalid utf8 byte sequences if it
-      # truncates a diff in the middle of a multibyte character. In this case
-      # CharlockHolmes will try to guess the encoding and will likely suggest an
-      # obscure encoding with low confidence.
-      # There is a lot more info with this merge request:
-      # https://gitlab.com/gitlab-org/gitlab_git/merge_requests/77#note_4754193
-      ENCODING_CONFIDENCE_THRESHOLD = 40
-
-      def encode!(message)
-        return nil unless message.respond_to? :force_encoding
-
-        # if message is utf-8 encoding, just return it
-        message.force_encoding("UTF-8")
-        return message if message.valid_encoding?
-
-        # return message if message type is binary
-        detect = CharlockHolmes::EncodingDetector.detect(message)
-        return message.force_encoding("BINARY") if detect && detect[:type] == :binary
-
-        # force detected encoding if we have sufficient confidence.
-        if detect && detect[:encoding] && detect[:confidence] > ENCODING_CONFIDENCE_THRESHOLD
-          message.force_encoding(detect[:encoding])
-        end
-
-        # encode and clean the bad chars
-        message.replace clean(message)
-      rescue
-        encoding = detect ? detect[:encoding] : "unknown"
-        "--broken encoding: #{encoding}"
-      end
-
-      def encode_utf8(message)
-        detect = CharlockHolmes::EncodingDetector.detect(message)
-        if detect
-          begin
-            CharlockHolmes::Converter.convert(message, detect[:encoding], 'UTF-8')
-          rescue ArgumentError => e
-            Rails.logger.warn("Ignoring error converting #{detect[:encoding]} into UTF8: #{e.message}")
-
-            ''
-          end
-        else
-          clean(message)
-        end
-      end
-
-      private
-
-      def clean(message)
-        message.encode("UTF-16BE", undef: :replace, invalid: :replace, replace: "")
-          .encode("UTF-8")
-          .gsub("\0".encode("UTF-8"), "")
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb
index 37ef6836742..ebf7393dc61 100644
--- a/lib/gitlab/git/ref.rb
+++ b/lib/gitlab/git/ref.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Git
     class Ref
-      include Gitlab::Git::EncodingHelper
+      include Gitlab::EncodingHelper
 
       # Branch or tag name
       # without "refs/tags|heads" prefix
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index d41256d9a84..b9afa05c819 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -1,7 +1,7 @@
 module Gitlab
   module Git
     class Tree
-      include Gitlab::Git::EncodingHelper
+      include Gitlab::EncodingHelper
 
       attr_accessor :id, :root_id, :name, :path, :type,
         :mode, :commit_id, :submodule_url
diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb
index 4c395b4266e..fa182c4deda 100644
--- a/lib/gitlab/utils.rb
+++ b/lib/gitlab/utils.rb
@@ -21,5 +21,13 @@ module Gitlab
 
       nil
     end
+
+    def boolean_to_yes_no(bool)
+      if bool
+        'Yes'
+      else
+        'No'
+      end
+    end
   end
 end