diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-09 13:39:09 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-03-09 13:39:09 +0300 |
commit | 4780d61e24e2bdd797838d85f47cf3a9f75fe1aa (patch) | |
tree | 1477bc075c5520e6ecf4d17c87b0f7e64a8bd1c5 | |
parent | e2c9cd893bf749bee91aeb26be03e59dc479912b (diff) |
Add latest changes from gitlab-org/gitlab@14-7-stable-ee
7 files changed, 61 insertions, 32 deletions
diff --git a/db/migrate/20210811193033_add_unique_index_to_vulnerability_finding_links.rb b/db/migrate/20210811193033_add_unique_index_to_vulnerability_finding_links.rb index 1bcee89ae57..cf51bca630a 100644 --- a/db/migrate/20210811193033_add_unique_index_to_vulnerability_finding_links.rb +++ b/db/migrate/20210811193033_add_unique_index_to_vulnerability_finding_links.rb @@ -1,18 +1,14 @@ # frozen_string_literal: true class AddUniqueIndexToVulnerabilityFindingLinks < Gitlab::Database::Migration[1.0] - disable_ddl_transaction! - - NAME_URL_INDEX_NAME = 'finding_link_name_url_idx' - URL_INDEX_NAME = 'finding_link_url_idx' + # This migration has been moved to db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb + # Previously, this was causing an bug where there was a conflict between the table cleanup and the index creation. def up - add_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :name, :url], unique: true, name: NAME_URL_INDEX_NAME - add_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :url], unique: true, where: 'name is null', name: URL_INDEX_NAME + # no op end def down - remove_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :name, :url], name: NAME_URL_INDEX_NAME - remove_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :url], name: URL_INDEX_NAME + # no op end end diff --git a/db/post_migrate/20211104165220_remove_vulnerability_finding_links.rb b/db/post_migrate/20211104165220_remove_vulnerability_finding_links.rb index fc50aa812a7..1625d24ef92 100644 --- a/db/post_migrate/20211104165220_remove_vulnerability_finding_links.rb +++ b/db/post_migrate/20211104165220_remove_vulnerability_finding_links.rb @@ -1,21 +1,14 @@ # frozen_string_literal: true class RemoveVulnerabilityFindingLinks < Gitlab::Database::Migration[1.0] - BATCH_SIZE = 50_000 - MIGRATION = 'RemoveVulnerabilityFindingLinks' - - disable_ddl_transaction! + # This migration has been moved to a TRUNCATE in db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb + # Previously, this was causing an bug where there was a conflict between the table cleanup and the index creation. def up - queue_background_migration_jobs_by_range_at_intervals( - define_batchable_model('vulnerability_finding_links'), - MIGRATION, - 2.minutes, - batch_size: BATCH_SIZE - ) + # no op end def down - # no ops + # no op end end diff --git a/db/post_migrate/20211210173137_remove_vulnerability_finding_links_again.rb b/db/post_migrate/20211210173137_remove_vulnerability_finding_links_again.rb index 98ac4433193..ffe8bf0d794 100644 --- a/db/post_migrate/20211210173137_remove_vulnerability_finding_links_again.rb +++ b/db/post_migrate/20211210173137_remove_vulnerability_finding_links_again.rb @@ -1,21 +1,14 @@ # frozen_string_literal: true class RemoveVulnerabilityFindingLinksAgain < Gitlab::Database::Migration[1.0] - BATCH_SIZE = 50_000 - MIGRATION = 'RemoveVulnerabilityFindingLinks' - - disable_ddl_transaction! + # This migration has been moved to a TRUNCATE in db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb + # Previously, this was causing an bug where there was a conflict between the table cleanup and the index creation. def up - queue_background_migration_jobs_by_range_at_intervals( - define_batchable_model('vulnerability_finding_links'), - MIGRATION, - 2.minutes, - batch_size: BATCH_SIZE - ) + # no op end def down - # no ops + # no op end end diff --git a/db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb b/db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb new file mode 100644 index 00000000000..cc9dabdf624 --- /dev/null +++ b/db/post_migrate/20220201193033_add_unique_index_to_vulnerability_finding_links_with_truncate.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddUniqueIndexToVulnerabilityFindingLinksWithTruncate < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + NAME_URL_INDEX_NAME = 'finding_link_name_url_idx' + URL_INDEX_NAME = 'finding_link_url_idx' + + def up + execute('TRUNCATE TABLE vulnerability_finding_links') + + add_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :name, :url], unique: true, name: NAME_URL_INDEX_NAME + add_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :url], unique: true, where: 'name is null', name: URL_INDEX_NAME + end + + def down + remove_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :name, :url], name: NAME_URL_INDEX_NAME + remove_concurrent_index :vulnerability_finding_links, [:vulnerability_occurrence_id, :url], name: URL_INDEX_NAME + end +end diff --git a/db/schema_migrations/20220201193033 b/db/schema_migrations/20220201193033 new file mode 100644 index 00000000000..e40840c66fa --- /dev/null +++ b/db/schema_migrations/20220201193033 @@ -0,0 +1 @@ +92bbe74c6c3627dd26f709acd2a20f442212eab933f719be815701a3bc429539
\ No newline at end of file diff --git a/lib/gitlab/cleanup/orphan_job_artifact_files.rb b/lib/gitlab/cleanup/orphan_job_artifact_files.rb index 05dfdcd4486..90123b9d000 100644 --- a/lib/gitlab/cleanup/orphan_job_artifact_files.rb +++ b/lib/gitlab/cleanup/orphan_job_artifact_files.rb @@ -99,6 +99,9 @@ module Gitlab # ^--+--+- components of hashed storage project path cmd += %w[-mindepth 6 -maxdepth 6] + # Intentionally exclude pipeline artifacts which match the same path + cmd += %w[-not -path */pipelines/*] + # Artifact directories are named on their ID cmd += %w[-type d] diff --git a/spec/lib/gitlab/cleanup/orphan_job_artifact_files_spec.rb b/spec/lib/gitlab/cleanup/orphan_job_artifact_files_spec.rb index b5adb603dab..e6ef2d8a541 100644 --- a/spec/lib/gitlab/cleanup/orphan_job_artifact_files_spec.rb +++ b/spec/lib/gitlab/cleanup/orphan_job_artifact_files_spec.rb @@ -34,10 +34,33 @@ RSpec.describe Gitlab::Cleanup::OrphanJobArtifactFiles do cleanup.run! end - it 'finds artifacts on disk' do + it 'finds job artifacts on disk' do artifact = create(:ci_job_artifact, :archive) + artifact_directory = artifact.file.relative_path.to_s.split('/')[0...6].join('/') + + cleaned = [] + + expect(cleanup).to receive(:find_artifacts).and_wrap_original do |original_method, *args, &block| + original_method.call(*args) { |dir| cleaned << dir } + end + + cleanup.run! + + expect(cleaned).to include(/#{artifact_directory}/) + end + + it 'does not find pipeline artifacts on disk' do + artifact = create(:ci_pipeline_artifact, :with_coverage_report) + # using 0...6 to match the -min/maxdepth 6 strictly, since this is one directory + # deeper than job artifacts, and .dirname would not match + artifact_directory = artifact.file.relative_path.to_s.split('/')[0...6].join('/') + + expect(cleanup).to receive(:find_artifacts).and_wrap_original do |original_method, *args, &block| + # this can either _not_ yield at all, or yield with any other file + # except the one that we're explicitly excluding + original_method.call(*args) { |path| expect(path).not_to match(artifact_directory) } + end - expect(cleanup).to receive(:find_artifacts).and_yield(artifact.file.path) cleanup.run! end |