diff options
Diffstat (limited to 'lib')
-rw-r--r-- | lib/api/internal/pages.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml | 2 | ||||
-rw-r--r-- | lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml | 1 | ||||
-rw-r--r-- | lib/gitlab/diff/deprecated_highlight_cache.rb | 68 | ||||
-rw-r--r-- | lib/gitlab/diff/file_collection/merge_request_diff.rb | 6 | ||||
-rw-r--r-- | lib/gitlab/diff/highlight_cache.rb | 93 | ||||
-rw-r--r-- | lib/gitlab/diff/line.rb | 3 |
7 files changed, 152 insertions, 23 deletions
diff --git a/lib/api/internal/pages.rb b/lib/api/internal/pages.rb index 003af7f6dd4..a2fe3e09df8 100644 --- a/lib/api/internal/pages.rb +++ b/lib/api/internal/pages.rb @@ -25,7 +25,7 @@ module API end get "/" do host = Namespace.find_by_pages_host(params[:host]) || PagesDomain.find_by_domain(params[:host]) - not_found! unless host + no_content! unless host virtual_domain = host.pages_virtual_domain no_content! unless virtual_domain diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index ef2fc561201..f708e95c2cf 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/container_scanning/ variables: - CS_MAJOR_VERSION: 1 + CS_MAJOR_VERSION: 2 container_scanning: stage: test diff --git a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml index 3f9d61a79f5..a37714eeed3 100644 --- a/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Dependency-Scanning.gitlab-ci.yml @@ -43,7 +43,6 @@ dependency_scanning: DS_ANALYZER_IMAGE_TAG \ DS_DEFAULT_ANALYZERS \ DS_EXCLUDED_PATHS \ - DEP_SCAN_DISABLE_REMOTE_CHECKS \ DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \ DS_PULL_ANALYZER_IMAGE_TIMEOUT \ DS_RUN_ANALYZER_TIMEOUT \ diff --git a/lib/gitlab/diff/deprecated_highlight_cache.rb b/lib/gitlab/diff/deprecated_highlight_cache.rb new file mode 100644 index 00000000000..47347686973 --- /dev/null +++ b/lib/gitlab/diff/deprecated_highlight_cache.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true +# +module Gitlab + module Diff + class DeprecatedHighlightCache + delegate :diffable, to: :@diff_collection + delegate :diff_options, to: :@diff_collection + + def initialize(diff_collection, backend: Rails.cache) + @backend = backend + @diff_collection = diff_collection + end + + # - Reads from cache + # - Assigns DiffFile#highlighted_diff_lines for cached files + def decorate(diff_file) + if content = read_file(diff_file) + diff_file.highlighted_diff_lines = content.map do |line| + Gitlab::Diff::Line.init_from_hash(line) + end + end + end + + # It populates a Hash in order to submit a single write to the memory + # cache. This avoids excessive IO generated by N+1's (1 writing for + # each highlighted line or file). + def write_if_empty + return if cached_content.present? + + @diff_collection.diff_files.each do |diff_file| + next unless cacheable?(diff_file) + + diff_file_id = diff_file.file_identifier + + cached_content[diff_file_id] = diff_file.highlighted_diff_lines.map(&:to_hash) + end + + cache.write(key, cached_content, expires_in: 1.week) + end + + def clear + cache.delete(key) + end + + def key + [diffable, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options] + end + + private + + def read_file(diff_file) + cached_content[diff_file.file_identifier] + end + + def cache + @backend + end + + def cached_content + @cached_content ||= cache.read(key) || {} + end + + def cacheable?(diff_file) + diffable.present? && diff_file.text? && diff_file.diffable? + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/merge_request_diff.rb b/lib/gitlab/diff/file_collection/merge_request_diff.rb index 3d661111f13..1d99349304b 100644 --- a/lib/gitlab/diff/file_collection/merge_request_diff.rb +++ b/lib/gitlab/diff/file_collection/merge_request_diff.rb @@ -37,7 +37,11 @@ module Gitlab private def cache - @cache ||= Gitlab::Diff::HighlightCache.new(self) + @cache ||= if Feature.enabled?(:hset_redis_diff_caching, project) + Gitlab::Diff::HighlightCache.new(self) + else + Gitlab::Diff::DeprecatedHighlightCache.new(self) + end end end end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index e4390771db2..12a1db70b36 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -3,16 +3,19 @@ module Gitlab module Diff class HighlightCache - delegate :diffable, to: :@diff_collection + EXPIRATION = 1.week + VERSION = 1 + + delegate :diffable, to: :@diff_collection delegate :diff_options, to: :@diff_collection - def initialize(diff_collection, backend: Rails.cache) - @backend = backend + def initialize(diff_collection) @diff_collection = diff_collection end # - Reads from cache # - Assigns DiffFile#highlighted_diff_lines for cached files + # def decorate(diff_file) if content = read_file(diff_file) diff_file.highlighted_diff_lines = content.map do |line| @@ -21,43 +24,95 @@ module Gitlab end end - # It populates a Hash in order to submit a single write to the memory - # cache. This avoids excessive IO generated by N+1's (1 writing for - # each highlighted line or file). + # For every file that isn't already contained in the redis hash, store the + # result of #highlighted_diff_lines, then submit the uncached content + # to #write_to_redis_hash to submit a single write. This avoids excessive + # IO generated by N+1's (1 writing for each highlighted line or file). + # def write_if_empty - return if cached_content.present? + return if uncached_files.empty? - @diff_collection.diff_files.each do |diff_file| + new_cache_content = {} + uncached_files.each do |diff_file| next unless cacheable?(diff_file) - diff_file_id = diff_file.file_identifier - - cached_content[diff_file_id] = diff_file.highlighted_diff_lines.map(&:to_hash) + new_cache_content[diff_file.file_path] = diff_file.highlighted_diff_lines.map(&:to_hash) end - cache.write(key, cached_content, expires_in: 1.week) + write_to_redis_hash(new_cache_content) end def clear - cache.delete(key) + Gitlab::Redis::Cache.with do |redis| + redis.del(key) + end end def key - [diffable, 'highlighted-diff-files', Gitlab::Diff::Line::SERIALIZE_KEYS, diff_options] + @redis_key ||= ['highlighted-diff-files', diffable.cache_key, VERSION, diff_options].join(":") end private - def read_file(diff_file) - cached_content[diff_file.file_identifier] + def uncached_files + diff_files = @diff_collection.diff_files + + diff_files.select { |file| read_cache[file.file_path].nil? } end - def cache - @backend + # Given a hash of: + # { "file/to/cache" => + # [ { line_code: "a5cc2925ca8258af241be7e5b0381edf30266302_19_19", + # rich_text: " <span id=\"LC19\" class=\"line\" lang=\"plaintext\">config/initializers/secret_token.rb</span>\n", + # text: " config/initializers/secret_token.rb", + # type: nil, + # index: 3, + # old_pos: 19, + # new_pos: 19 } + # ] } + # + # ...it will write/update a Gitlab::Redis hash (HSET) + # + def write_to_redis_hash(hash) + Gitlab::Redis::Cache.with do |redis| + redis.pipelined do + hash.each do |diff_file_id, highlighted_diff_lines_hash| + redis.hset(key, diff_file_id, highlighted_diff_lines_hash.to_json) + end + + # HSETs have to have their expiration date manually updated + # + redis.expire(key, EXPIRATION) + end + end + end + + def file_paths + @file_paths ||= @diff_collection.diffs.collect(&:file_path) + end + + def read_file(diff_file) + cached_content[diff_file.file_path] end def cached_content - @cached_content ||= cache.read(key) || {} + @cached_content ||= read_cache + end + + def read_cache + return {} unless file_paths.any? + + results = [] + + Gitlab::Redis::Cache.with do |redis| + results = redis.hmget(key, file_paths) + end + + results.map! do |result| + JSON.parse(result, symbolize_names: true) unless result.nil? + end + + file_paths.zip(results).to_h end def cacheable?(diff_file) diff --git a/lib/gitlab/diff/line.rb b/lib/gitlab/diff/line.rb index 001748afb41..28ea1921f90 100644 --- a/lib/gitlab/diff/line.rb +++ b/lib/gitlab/diff/line.rb @@ -3,6 +3,9 @@ module Gitlab module Diff class Line + # When SERIALIZE_KEYS is updated, to reset the redis cache entries you'll + # need to bump the VERSION constant on Gitlab::Diff::HighlightCache + # SERIALIZE_KEYS = %i(line_code rich_text text type index old_pos new_pos).freeze attr_reader :line_code, :type, :old_pos, :new_pos |