Add latest changes from gitlab-org/gitlab@15-1-stable-eev15.1.0-rc42

5 files changed, 139 insertions, 157 deletions

diff --git a/lib/gitlab/diff/custom_diff.rb b/lib/gitlab/diff/custom_diff.rb
deleted file mode 100644
index 860f87a28a3..00000000000
--- a/lib/gitlab/diff/custom_diff.rb
+++ /dev/null
@@ -1,89 +0,0 @@
-# frozen_string_literal: true
-module Gitlab
-  module Diff
-    module CustomDiff
-      RENDERED_TIMEOUT_BACKGROUND = 20.seconds
-      RENDERED_TIMEOUT_FOREGROUND = 1.5.seconds
-      BACKGROUND_EXECUTION = 'background'
-      FOREGROUND_EXECUTION = 'foreground'
-      LOG_IPYNBDIFF_GENERATED = 'IPYNB_DIFF_GENERATED'
-      LOG_IPYNBDIFF_TIMEOUT = 'IPYNB_DIFF_TIMEOUT'
-      LOG_IPYNBDIFF_INVALID = 'IPYNB_DIFF_INVALID'
-
-      class << self
-        def preprocess_before_diff(path, old_blob, new_blob)
-          return unless path.ends_with? '.ipynb'
-
-          Timeout.timeout(timeout_time) do
-            transformed_diff(old_blob&.data, new_blob&.data)&.tap do
-              transformed_for_diff(new_blob, old_blob)
-              log_event(LOG_IPYNBDIFF_GENERATED)
-            end
-          end
-        rescue Timeout::Error => e
-          rendered_timeout.increment(source: execution_source)
-          log_event(LOG_IPYNBDIFF_TIMEOUT, e)
-        rescue IpynbDiff::InvalidNotebookError, IpynbDiff::InvalidTokenError => e
-          log_event(LOG_IPYNBDIFF_INVALID, e)
-        end
-
-        def transformed_diff(before, after)
-          transformed_diff = IpynbDiff.diff(before, after,
-                         raise_if_invalid_nb: true,
-                         diffy_opts: { include_diff_info: true }).to_s(:text)
-
-          strip_diff_frontmatter(transformed_diff)
-        end
-
-        def transformed_blob_language(blob)
-          'md' if transformed_for_diff?(blob)
-        end
-
-        def transformed_blob_data(blob)
-          if transformed_for_diff?(blob)
-            IpynbDiff.transform(blob.data, raise_errors: true, include_frontmatter: false)
-          end
-        end
-
-        def strip_diff_frontmatter(diff_content)
-          diff_content.scan(/.*\n/)[2..]&.join('') if diff_content.present?
-        end
-
-        def blobs_with_transformed_diffs
-          @blobs_with_transformed_diffs ||= {}
-        end
-
-        def transformed_for_diff?(blob)
-          blobs_with_transformed_diffs[blob]
-        end
-
-        def transformed_for_diff(*blobs)
-          blobs.each do |b|
-            blobs_with_transformed_diffs[b] = true if b
-          end
-        end
-
-        def rendered_timeout
-          @rendered_timeout ||= Gitlab::Metrics.counter(
-            :ipynb_semantic_diff_timeouts_total,
-            'Counts the times notebook rendering timed out'
-          )
-        end
-
-        def timeout_time
-          Gitlab::Runtime.sidekiq? ? RENDERED_TIMEOUT_BACKGROUND : RENDERED_TIMEOUT_FOREGROUND
-        end
-
-        def execution_source
-          Gitlab::Runtime.sidekiq? ? BACKGROUND_EXECUTION : FOREGROUND_EXECUTION
-        end
-
-        def log_event(message, error = nil)
-          Gitlab::AppLogger.info({ message: message })
-          Gitlab::ErrorTracking.track_exception(error) if error
-          nil
-        end
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb
index d6ee21b93b6..8e039d32ef5 100644
--- a/lib/gitlab/diff/file.rb
+++ b/lib/gitlab/diff/file.rb
@@ -43,20 +43,12 @@ module Gitlab
         # Ensure items are collected in the the batch
         new_blob_lazy
         old_blob_lazy
-
-        if use_semantic_ipynb_diff? && !use_renderable_diff?
-          diff.diff = Gitlab::Diff::CustomDiff.preprocess_before_diff(diff.new_path, old_blob_lazy, new_blob_lazy) || diff.diff
-        end
       end
 
       def use_semantic_ipynb_diff?
         strong_memoize(:_use_semantic_ipynb_diff) { Feature.enabled?(:ipynb_semantic_diff, repository.project) }
       end
 
-      def use_renderable_diff?
-        strong_memoize(:_renderable_diff_enabled) { Feature.enabled?(:rendered_diffs_viewer, repository.project) }
-      end
-
       def has_renderable?
         rendered&.has_renderable?
       end
@@ -381,7 +373,7 @@ module Gitlab
       end
 
       def rendered
-        return unless use_semantic_ipynb_diff? && use_renderable_diff? && ipynb? && modified_file? && !too_large?
+        return unless use_semantic_ipynb_diff? && ipynb? && modified_file? && !too_large?
 
         strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) }
       end
diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb
index 316a0d2815a..75127098600 100644
--- a/lib/gitlab/diff/line.rb
+++ b/lib/gitlab/diff/line.rb
@@ -10,7 +10,7 @@ module Gitlab
 
       attr_reader :marker_ranges
       attr_writer :text, :rich_text
-      attr_accessor :index, :old_pos, :new_pos, :line_code, :type
+      attr_accessor :index, :old_pos, :new_pos, :line_code, :type, :embedded_image
 
       def initialize(text, type, index, old_pos, new_pos, parent_file: nil, line_code: nil, rich_text: nil)
         @text = text
diff --git a/lib/gitlab/diff/rendered/notebook/diff_file.rb b/lib/gitlab/diff/rendered/notebook/diff_file.rb
index 1f064d8af50..0a5b2ec3890 100644
--- a/lib/gitlab/diff/rendered/notebook/diff_file.rb
+++ b/lib/gitlab/diff/rendered/notebook/diff_file.rb
@@ -3,11 +3,11 @@ module Gitlab
   module Diff
     module Rendered
       module Notebook
-        include Gitlab::Utils::StrongMemoize
-
         class DiffFile < Gitlab::Diff::File
+          include Gitlab::Diff::Rendered::Notebook::DiffFileHelper
+          include Gitlab::Utils::StrongMemoize
+
           RENDERED_TIMEOUT_BACKGROUND = 10.seconds
-          RENDERED_TIMEOUT_FOREGROUND = 1.5.seconds
           BACKGROUND_EXECUTION = 'background'
           FOREGROUND_EXECUTION = 'foreground'
           LOG_IPYNBDIFF_GENERATED = 'IPYNB_DIFF_GENERATED'
@@ -49,11 +49,14 @@ module Gitlab
           end
 
           def highlighted_diff_lines
-            @highlighted_diff_lines ||= begin
-              removal_line_maps, addition_line_maps = compute_end_start_map
-              Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight.map do |line|
-                mutate_line(line, addition_line_maps, removal_line_maps)
-              end
+            strong_memoize(:highlighted_diff_lines) do
+              lines = Gitlab::Diff::Highlight.new(self, repository: self.repository).highlight
+              lines_in_source = lines_in_source_diff(
+                source_diff.highlighted_diff_lines, source_diff.deleted_file?, source_diff.new_file?
+              )
+
+              lines.zip(line_positions_at_source_diff(lines, transformed_blocks))
+                   .map { |line, positions| mutate_line(line, positions, lines_in_source)}
             end
           end
 
@@ -66,10 +69,9 @@ module Gitlab
                 next
               end
 
-              Timeout.timeout(timeout_time) do
+              Gitlab::RenderTimeout.timeout(background: RENDERED_TIMEOUT_BACKGROUND) do
                 IpynbDiff.diff(source_diff.old_blob&.data, source_diff.new_blob&.data,
                                raise_if_invalid_nb: true,
-                               hide_images: true,
                                diffy_opts: { include_diff_info: true })&.tap do
                   log_event(LOG_IPYNBDIFF_GENERATED)
                 end
@@ -90,50 +92,8 @@ module Gitlab
             diff
           end
 
-          def strip_diff_frontmatter(diff_content)
-            diff_content.scan(/.*\n/)[2..]&.join('') if diff_content.present?
-          end
-
-          def transformed_line_to_source(transformed_line, transformed_blocks)
-            transformed_blocks.empty? ? 0 : ( transformed_blocks[transformed_line - 1][:source_line] || -1 ) + 1
-          end
-
-          def mutate_line(line, addition_line_maps, removal_line_maps)
-            line.new_pos = transformed_line_to_source(line.new_pos, notebook_diff.to.blocks)
-            line.old_pos = transformed_line_to_source(line.old_pos, notebook_diff.from.blocks)
-
-            line.old_pos = addition_line_maps[line.new_pos] if line.old_pos == 0 && line.new_pos != 0
-            line.new_pos = removal_line_maps[line.old_pos] if line.new_pos == 0 && line.old_pos != 0
-
-            # Lines that do not appear on the original diff should not be commentable
-            line.type = "#{line.type || 'unchanged'}-nomappinginraw" unless addition_line_maps[line.new_pos] || removal_line_maps[line.old_pos]
-
-            line.line_code = line_code(line)
-            line
-          end
-
-          def compute_end_start_map
-            # line_codes are used for assigning notes to diffs, and these depend on the line on the new version and the
-            # line that would have been that one in the previous version. However, since we do a transformation on the
-            # file, that map gets lost. To overcome this, we look at the original source lines and build two maps:
-            # - For additions, we look at the latest line change for that line and pick the old line for that id
-            # - For removals, we look at the first line in the old version, and pick the first line on the new version
-            #
-            #
-            # The caveat here is that we can't have notes on lines that are not a translation of a line in the source
-            # diff
-            #
-            # (gitlab/diff/file.rb:75)
-
-            removals = {}
-            additions = {}
-
-            source_diff.highlighted_diff_lines.each do |line|
-              removals[line.old_pos] = line.new_pos unless source_diff.new_file?
-              additions[line.new_pos] = line.old_pos unless source_diff.deleted_file?
-            end
-
-            [removals, additions]
+          def transformed_blocks
+            { from: notebook_diff.from.blocks, to: notebook_diff.to.blocks }
           end
 
           def rendered_timeout
@@ -143,15 +103,26 @@ module Gitlab
             )
           end
 
-          def timeout_time
-            Gitlab::Runtime.sidekiq? ? RENDERED_TIMEOUT_BACKGROUND : RENDERED_TIMEOUT_FOREGROUND
-          end
-
           def log_event(message, error = nil)
             Gitlab::AppLogger.info({ message: message })
             Gitlab::ErrorTracking.log_exception(error) if error
             nil
           end
+
+          def mutate_line(line, mapped_positions, source_diff_lines)
+            line.old_pos, line.new_pos = mapped_positions
+
+            # Lines that do not appear on the original diff should not be commentable
+            unless source_diff_lines[:to].include?(line.new_pos) || source_diff_lines[:from].include?(line.old_pos)
+              line.type = "#{line.type || 'unchanged'}-nomappinginraw"
+            end
+
+            line.line_code = line_code(line)
+
+            line.rich_text = image_as_rich_text(line.text) || line.rich_text
+
+            line
+          end
         end
       end
     end
diff --git a/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb b/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb
new file mode 100644
index 00000000000..2e1b5ea301d
--- /dev/null
+++ b/lib/gitlab/diff/rendered/notebook/diff_file_helper.rb
@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+module Gitlab
+  module Diff
+    module Rendered
+      module Notebook
+        module DiffFileHelper
+          require 'set'
+
+          EMBEDDED_IMAGE_PATTERN = '    ![](data:image'
+
+          def strip_diff_frontmatter(diff_content)
+            diff_content.scan(/.*\n/)[2..]&.join('') if diff_content.present?
+          end
+
+          # line_positions_at_source_diff: given the transformed lines,
+          # what are the correct values for old_pos and new_pos?
+          #
+          # Example:
+          #
+          # Original
+          # from | to
+          # A    | A
+          # B    | D
+          # C    | E
+          # F    | F
+          #
+          # Original Diff
+          #   A A
+          # - B
+          # - C
+          # +   D
+          # +   E
+          #   F F
+          #
+          # Transformed
+          # from | to
+          # A    | A
+          # C    | D
+          # B    | J
+          # L    | E
+          # K    | K
+          # F    | F
+          #
+          # Transformed diff | transf old, new | OG old_pos, new_pos |
+          #  A A             | 1, 1            | 1, 1                |
+          # -C               | 2, 2            | 3, 2                |
+          # -B               | 3, 2            | 2, 2                |
+          # -L               | 4, 2            | 0, 0                |
+          # +  D             | 5, 2            | 4, 2                |
+          # +  J             | 5, 3            | 0, 0                |
+          # +  E             | 5, 4            | 4, 3                |
+          #  K K             | 5, 5            | 0, 0                |
+          #  F F             | 6, 6            | 4, 4                |
+          def line_positions_at_source_diff(lines, blocks)
+            last_mapped_old_pos = 0
+            last_mapped_new_pos = 0
+
+            lines.reverse_each.map do |line|
+              old_pos = source_line_from_block(line.old_pos, blocks[:from])
+              new_pos = source_line_from_block(line.new_pos, blocks[:to])
+
+              old_has_no_mapping = old_pos == 0
+              new_has_no_mapping = new_pos == 0
+
+              next [0, 0] if old_has_no_mapping && (new_has_no_mapping || line.type == 'old')
+              next [0, 0] if new_has_no_mapping && line.type == 'new'
+
+              new_pos = last_mapped_new_pos if new_has_no_mapping && line.type == 'old'
+              old_pos = last_mapped_old_pos if old_has_no_mapping && line.type == 'new'
+
+              last_mapped_old_pos = old_pos
+              last_mapped_new_pos = new_pos
+
+              [old_pos, new_pos]
+            end.reverse
+          end
+
+          def lines_in_source_diff(source_diff_lines, is_deleted_file, is_added_file)
+            {
+              from: is_added_file ? Set[] : source_diff_lines.map {|l| l.old_pos}.to_set,
+              to: is_deleted_file ? Set[] : source_diff_lines.map {|l| l.new_pos}.to_set
+            }
+          end
+
+          def source_line_from_block(transformed_line, transformed_blocks)
+            # Blocks are the lines returned from the library and are a hash with {text:, source_line:}
+            # Blocks source_line are 0 indexed
+            return 0 if transformed_blocks.empty?
+
+            line_in_source = transformed_blocks[transformed_line - 1][:source_line]
+
+            return 0 unless line_in_source.present?
+
+            line_in_source + 1
+          end
+
+          def image_as_rich_text(line_text)
+            return unless line_text[1..].starts_with?(EMBEDDED_IMAGE_PATTERN)
+
+            image_body = line_text[1..].delete_prefix(EMBEDDED_IMAGE_PATTERN).delete_suffix(')')
+
+            "<img src=\"data:image#{CGI.escapeHTML(image_body)}\">".html_safe
+          end
+        end
+      end
+    end
+  end
+end