Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/gems
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-12-19 14:01:45 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-12-19 14:01:45 +0300
commit9297025d0b7ddf095eb618dfaaab2ff8f2018d8b (patch)
tree865198c01d1824a9b098127baa3ab980c9cd2c06 /gems
parent6372471f43ee03c05a7c1f8b0c6ac6b8a7431dbe (diff)
Add latest changes from gitlab-org/gitlab@16-7-stable-eev16.7.0-rc42
Diffstat (limited to 'gems')
-rw-r--r--gems/click_house-client/lib/click_house/client.rb4
-rw-r--r--gems/csv_builder/lib/csv_builder/builder.rb2
-rw-r--r--gems/csv_builder/spec/csv_builder_spec.rb128
-rw-r--r--gems/gem-pg.gitlab-ci.yml80
-rw-r--r--gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb1
-rw-r--r--gems/gitlab-backup-cli/lib/gitlab/backup/cli/utils.rb11
-rw-r--r--gems/gitlab-backup-cli/lib/gitlab/backup/cli/utils/pg_dump.rb59
-rw-r--r--gems/gitlab-backup-cli/sig/gitlab/backup/cli.rbs1
-rw-r--r--gems/gitlab-backup-cli/sig/gitlab/backup/cli/utils/pg_dump.rbs25
-rw-r--r--gems/gitlab-backup-cli/spec/gitlab/backup/cli/utils/pg_dump_spec.rb81
-rw-r--r--gems/gitlab-database-load_balancing/.gitignore (renamed from gems/rspec_flaky/.gitignore)0
-rw-r--r--gems/gitlab-database-load_balancing/.gitlab-ci.yml4
-rw-r--r--gems/gitlab-database-load_balancing/.rspec (renamed from gems/rspec_flaky/.rspec)0
-rw-r--r--gems/gitlab-database-load_balancing/.rubocop.yml6
-rw-r--r--gems/gitlab-database-load_balancing/Gemfile14
-rw-r--r--gems/gitlab-database-load_balancing/Gemfile.lock303
-rw-r--r--gems/gitlab-database-load_balancing/README.md3
-rw-r--r--gems/gitlab-database-load_balancing/Rakefile12
-rw-r--r--gems/gitlab-database-load_balancing/gitlab-database-load_balancing.gemspec31
-rw-r--r--gems/gitlab-database-load_balancing/spec/spec_helper.rb28
-rw-r--r--gems/gitlab-database-lock_retries/.gitignore11
-rw-r--r--gems/gitlab-database-lock_retries/.gitlab-ci.yml4
-rw-r--r--gems/gitlab-database-lock_retries/.rspec3
-rw-r--r--gems/gitlab-database-lock_retries/.rubocop.yml2
-rw-r--r--gems/gitlab-database-lock_retries/CHANGELOG.md5
-rw-r--r--gems/gitlab-database-lock_retries/Gemfile6
-rw-r--r--gems/gitlab-database-lock_retries/Gemfile.lock105
-rw-r--r--gems/gitlab-database-lock_retries/README.md29
-rw-r--r--gems/gitlab-database-lock_retries/Rakefile12
-rw-r--r--gems/gitlab-database-lock_retries/gitlab-database-lock_retries.gemspec23
-rw-r--r--gems/gitlab-database-lock_retries/lib/gitlab/database/lock_retries.rb11
-rw-r--r--gems/gitlab-database-lock_retries/lib/gitlab/database/lock_retries/version.rb9
-rw-r--r--gems/gitlab-database-lock_retries/spec/gitlab/database/lock_retries_spec.rb9
-rw-r--r--gems/gitlab-database-lock_retries/spec/spec_helper.rb15
-rw-r--r--gems/gitlab-housekeeper/.gitignore12
-rw-r--r--gems/gitlab-housekeeper/.gitlab-ci.yml (renamed from gems/rspec_flaky/.gitlab-ci.yml)2
-rw-r--r--gems/gitlab-housekeeper/.rspec3
-rw-r--r--gems/gitlab-housekeeper/.rubocop.yml14
-rw-r--r--gems/gitlab-housekeeper/Gemfile16
-rw-r--r--gems/gitlab-housekeeper/Gemfile.lock139
-rw-r--r--gems/gitlab-housekeeper/README.md34
-rw-r--r--gems/gitlab-housekeeper/Rakefile8
-rwxr-xr-xgems/gitlab-housekeeper/bin/gitlab-housekeeper32
-rw-r--r--gems/gitlab-housekeeper/gitlab-housekeeper.gemspec29
-rw-r--r--gems/gitlab-housekeeper/lib/gitlab/housekeeper.rb11
-rw-r--r--gems/gitlab-housekeeper/lib/gitlab/housekeeper/git.rb81
-rw-r--r--gems/gitlab-housekeeper/lib/gitlab/housekeeper/gitlab_client.rb118
-rw-r--r--gems/gitlab-housekeeper/lib/gitlab/housekeeper/keep.rb29
-rw-r--r--gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb102
-rw-r--r--gems/gitlab-housekeeper/lib/gitlab/housekeeper/shell.rb27
-rw-r--r--gems/gitlab-housekeeper/lib/gitlab/housekeeper/version.rb (renamed from gems/rspec_flaky/lib/rspec_flaky/version.rb)4
-rw-r--r--gems/gitlab-housekeeper/spec/gitlab/housekeeper/git_spec.rb128
-rw-r--r--gems/gitlab-housekeeper/spec/gitlab/housekeeper/gitlab_client_spec.rb99
-rw-r--r--gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb104
-rw-r--r--gems/gitlab-housekeeper/spec/gitlab/housekeeper/shell_spec.rb24
-rw-r--r--gems/gitlab-housekeeper/spec/gitlab/housekeeper_spec.rb7
-rw-r--r--gems/gitlab-housekeeper/spec/spec_helper.rb20
-rw-r--r--gems/gitlab-http/Gemfile.lock8
-rw-r--r--gems/gitlab-http/lib/gitlab/http_v2/client.rb13
-rw-r--r--gems/gitlab-http/lib/gitlab/http_v2/url_blocker.rb9
-rw-r--r--gems/gitlab-http/spec/gitlab/http_v2/url_blocker_spec.rb13
-rw-r--r--gems/gitlab-rspec/Gemfile.lock6
-rw-r--r--gems/gitlab-rspec/gitlab-rspec.gemspec1
-rw-r--r--gems/gitlab-rspec/lib/gitlab/rspec/all.rb3
-rw-r--r--gems/gitlab-rspec/lib/gitlab/rspec/next_found_instance_of.rb66
-rw-r--r--gems/gitlab-rspec/lib/gitlab/rspec/next_instance_of.rb39
-rw-r--r--gems/gitlab-rspec/lib/gitlab/rspec/stub_rails.rb57
-rw-r--r--gems/gitlab-rspec_flaky/.gitignore11
-rw-r--r--gems/gitlab-rspec_flaky/.gitlab-ci.yml4
-rw-r--r--gems/gitlab-rspec_flaky/.rspec3
-rw-r--r--gems/gitlab-rspec_flaky/.rubocop.yml (renamed from gems/rspec_flaky/.rubocop.yml)0
-rw-r--r--gems/gitlab-rspec_flaky/Gemfile (renamed from gems/rspec_flaky/Gemfile)2
-rw-r--r--gems/gitlab-rspec_flaky/Gemfile.lock (renamed from gems/rspec_flaky/Gemfile.lock)26
-rw-r--r--gems/gitlab-rspec_flaky/gitlab-rspec_flaky.gemspec (renamed from gems/rspec_flaky/rspec_flaky.gemspec)8
-rw-r--r--gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky.rb (renamed from gems/rspec_flaky/lib/rspec_flaky.rb)5
-rw-r--r--gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/config.rb23
-rw-r--r--gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/example.rb70
-rw-r--r--gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_example.rb61
-rw-r--r--gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_examples_collection.rb36
-rw-r--r--gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/listener.rb72
-rw-r--r--gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/report.rb61
-rw-r--r--gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/version.rb9
-rw-r--r--gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/config_spec.rb (renamed from gems/rspec_flaky/spec/rspec_flaky/config_spec.rb)4
-rw-r--r--gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/example_spec.rb (renamed from gems/rspec_flaky/spec/rspec_flaky/example_spec.rb)4
-rw-r--r--gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/flaky_example_spec.rb (renamed from gems/rspec_flaky/spec/rspec_flaky/flaky_example_spec.rb)4
-rw-r--r--gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/flaky_examples_collection_spec.rb (renamed from gems/rspec_flaky/spec/rspec_flaky/flaky_examples_collection_spec.rb)4
-rw-r--r--gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/listener_spec.rb (renamed from gems/rspec_flaky/spec/rspec_flaky/listener_spec.rb)24
-rw-r--r--gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/report_spec.rb (renamed from gems/rspec_flaky/spec/rspec_flaky/report_spec.rb)20
-rw-r--r--gems/gitlab-rspec_flaky/spec/spec_helper.rb (renamed from gems/rspec_flaky/spec/spec_helper.rb)2
-rw-r--r--gems/gitlab-secret_detection/.gitignore12
-rw-r--r--gems/gitlab-secret_detection/.gitlab-ci.yml4
-rw-r--r--gems/gitlab-secret_detection/.rspec3
-rw-r--r--gems/gitlab-secret_detection/.rubocop.yml8
-rw-r--r--gems/gitlab-secret_detection/CHANGELOG.md5
-rw-r--r--gems/gitlab-secret_detection/Gemfile6
-rw-r--r--gems/gitlab-secret_detection/Gemfile.lock149
-rw-r--r--gems/gitlab-secret_detection/README.md3
-rw-r--r--gems/gitlab-secret_detection/gitlab-secret_detection.gemspec37
-rw-r--r--gems/gitlab-secret_detection/lib/gitlab/secret_detection.rb11
-rw-r--r--gems/gitlab-secret_detection/lib/gitlab/secret_detection/finding.rb41
-rw-r--r--gems/gitlab-secret_detection/lib/gitlab/secret_detection/response.rb28
-rw-r--r--gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb206
-rw-r--r--gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb16
-rw-r--r--gems/gitlab-secret_detection/lib/gitlab/secret_detection/version.rb7
-rw-r--r--gems/gitlab-secret_detection/lib/gitleaks.toml307
-rw-r--r--gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb222
-rw-r--r--gems/gitlab-secret_detection/spec/spec_helper.rb18
-rw-r--r--gems/gitlab-utils/Gemfile.lock6
-rw-r--r--gems/gitlab-utils/lib/gitlab/utils.rb19
-rw-r--r--gems/gitlab-utils/lib/gitlab/utils/all.rb1
-rw-r--r--gems/gitlab-utils/lib/gitlab/utils/system.rb172
-rw-r--r--gems/gitlab-utils/spec/gitlab/utils/system_spec.rb364
-rw-r--r--gems/gitlab-utils/spec/gitlab/utils_spec.rb29
-rw-r--r--gems/rspec_flaky/lib/rspec_flaky/config.rb21
-rw-r--r--gems/rspec_flaky/lib/rspec_flaky/example.rb68
-rw-r--r--gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb59
-rw-r--r--gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb34
-rw-r--r--gems/rspec_flaky/lib/rspec_flaky/listener.rb70
-rw-r--r--gems/rspec_flaky/lib/rspec_flaky/report.rb59
119 files changed, 4321 insertions, 412 deletions
diff --git a/gems/click_house-client/lib/click_house/client.rb b/gems/click_house-client/lib/click_house/client.rb
index 1ca3653c45f..499201a1aba 100644
--- a/gems/click_house-client/lib/click_house/client.rb
+++ b/gems/click_house-client/lib/click_house/client.rb
@@ -30,6 +30,10 @@ module ClickHouse
DatabaseError = Class.new(Error)
QueryError = Class.new(Error)
+ def self.database_configured?(database, configuration = self.configuration)
+ !!configuration.databases[database]
+ end
+
# Executes a SELECT database query
def self.select(query, database, configuration = self.configuration)
instrumented_execute(query, database, configuration) do |response, instrument|
diff --git a/gems/csv_builder/lib/csv_builder/builder.rb b/gems/csv_builder/lib/csv_builder/builder.rb
index 99b63153ab2..51aaf2132cf 100644
--- a/gems/csv_builder/lib/csv_builder/builder.rb
+++ b/gems/csv_builder/lib/csv_builder/builder.rb
@@ -80,6 +80,8 @@ module CsvBuilder
attributes.map do |attribute|
if attribute.respond_to?(:call)
excel_sanitize(attribute.call(object))
+ elsif object.is_a?(Hash)
+ excel_sanitize(object[attribute])
else
excel_sanitize(object.public_send(attribute)) # rubocop:disable GitlabSecurity/PublicSend
end
diff --git a/gems/csv_builder/spec/csv_builder_spec.rb b/gems/csv_builder/spec/csv_builder_spec.rb
index 9d6283b3985..ccae8365174 100644
--- a/gems/csv_builder/spec/csv_builder_spec.rb
+++ b/gems/csv_builder/spec/csv_builder_spec.rb
@@ -1,18 +1,19 @@
# frozen_string_literal: true
RSpec.describe CsvBuilder do
- let(:object) { double(question: :answer) }
let(:csv_data) { subject.render }
+ let(:header_to_value_hash) do
+ { 'Q & A' => :question, 'Reversed' => ->(o) { o.question.to_s.reverse } }
+ end
let(:subject) do
- described_class.new(
- enumerable, 'Q & A' => :question, 'Reversed' => ->(o) { o.question.to_s.reverse })
+ described_class.new(enumerable, **header_to_value_hash)
end
shared_examples 'csv builder examples' do
let(:items) { [object] }
- it "has a version number" do
+ it 'has a version number' do
expect(CsvBuilder::Version::VERSION).not_to be nil
end
@@ -45,30 +46,6 @@ RSpec.describe CsvBuilder do
end
end
- describe 'truncation' do
- let(:big_object) { double(question: 'Long' * 1024) }
- let(:row_size) { big_object.question.length * 2 }
- let(:items) { [big_object, big_object, big_object] }
-
- it 'occurs after given number of bytes' do
- expect(subject.render(row_size * 2).length).to be_between(row_size * 2, row_size * 3)
- expect(subject).to be_truncated
- expect(subject.rows_written).to eq 2
- end
-
- it 'is ignored by default' do
- expect(subject.render.length).to be > row_size * 3
- expect(subject.rows_written).to eq 3
- end
-
- it 'causes rows_expected to fall back to .count' do
- subject.render(0)
-
- expect(enumerable).to receive(:count).and_call_original
- expect(subject.rows_expected).to eq 3
- end
- end
-
it 'avoids loading all data in a single query' do
expect(enumerable).to receive(:find_each)
@@ -83,42 +60,61 @@ RSpec.describe CsvBuilder do
expect(csv_data).to include 'answer'
end
- it 'allows lamdas to look up more complicated data' do
+ it 'allows lambdas to look up more complicated data' do
expect(csv_data).to include 'rewsna'
end
+ end
- describe 'excel sanitization' do
- let(:dangerous_title) { double(title: "=cmd|' /C calc'!A0 title", description: "*safe_desc") }
- let(:dangerous_desc) { double(title: "*safe_title", description: "=cmd|' /C calc'!A0 desc") }
- let(:items) { [dangerous_title, dangerous_desc] }
- let(:subject) { described_class.new(enumerable, 'Title' => 'title', 'Description' => 'description') }
- let(:csv_data) { subject.render }
+ shared_examples 'csv builder with truncation ability' do
+ let(:items) { [big_object, big_object, big_object] }
+ let(:row_size) { question_value.length * 2 }
- it 'sanitizes dangerous characters at the beginning of a column' do
- expect(csv_data).to include "'=cmd|' /C calc'!A0 title"
- expect(csv_data).to include "'=cmd|' /C calc'!A0 desc"
- end
+ it 'occurs after given number of bytes' do
+ expect(subject.render(row_size * 2).length).to be_between(row_size * 2, row_size * 3)
+ expect(subject).to be_truncated
+ expect(subject.rows_written).to eq 2
+ end
- it 'does not sanitize safe symbols at the beginning of a column' do
- expect(csv_data).not_to include "'*safe_desc"
- expect(csv_data).not_to include "'*safe_title"
- end
+ it 'is ignored by default' do
+ expect(subject.render.length).to be > row_size * 3
+ expect(subject.rows_written).to eq 3
+ end
- context 'when dangerous characters are after a line break' do
- let(:items) { [double(title: "Safe title", description: "With task list\n-[x] todo 1")] }
+ it 'causes rows_expected to fall back to .count' do
+ subject.render(0)
- it 'does not append single quote to description' do
- builder = described_class.new(enumerable, 'Title' => 'title', 'Description' => 'description')
+ expect(enumerable).to receive(:count).and_call_original
+ expect(subject.rows_expected).to eq 3
+ end
+ end
- csv_data = builder.render
+ shared_examples 'excel sanitization' do
+ let(:dangerous_title) { double(title: "=cmd|' /C calc'!A0 title", description: "*safe_desc") }
+ let(:dangerous_desc) { double(title: "*safe_title", description: "=cmd|' /C calc'!A0 desc") }
+ let(:header_to_value_hash) { { 'Title' => 'title', 'Description' => 'description' } }
+ let(:items) { [dangerous_title, dangerous_desc] }
- expect(csv_data).to eq("Title,Description\nSafe title,\"With task list\n-[x] todo 1\"\n")
- end
+ it 'sanitizes dangerous characters at the beginning of a column' do
+ expect(csv_data).to include "'=cmd|' /C calc'!A0 title"
+ expect(csv_data).to include "'=cmd|' /C calc'!A0 desc"
+ end
+
+ it 'does not sanitize safe symbols at the beginning of a column' do
+ expect(csv_data).not_to include "'*safe_desc"
+ expect(csv_data).not_to include "'*safe_title"
+ end
+
+ context 'when dangerous characters are after a line break' do
+ let(:items) { [double(title: "Safe title", description: "With task list\n-[x] todo 1")] }
+
+ it 'does not append single quote to description' do
+ expect(csv_data).to eq("Title,Description\nSafe title,\"With task list\n-[x] todo 1\"\n")
end
end
end
context 'when ActiveRecord::Relation like object is given' do
+ let(:object) { double(question: :answer) }
let(:enumerable) { described_class::FakeRelation.new(items) }
before do
@@ -132,11 +128,43 @@ RSpec.describe CsvBuilder do
end
it_behaves_like 'csv builder examples'
+ it_behaves_like 'excel sanitization'
+ it_behaves_like 'csv builder with truncation ability' do
+ let(:big_object) { double(question: 'Long' * 1024) }
+ let(:question_value) { big_object.question }
+ end
end
context 'when Enumerable like object is given' do
+ let(:object) { double(question: :answer) }
let(:enumerable) { items }
it_behaves_like 'csv builder examples'
+ it_behaves_like 'excel sanitization'
+ it_behaves_like 'csv builder with truncation ability' do
+ let(:big_object) { double(question: 'Long' * 1024) }
+ let(:question_value) { big_object.question }
+ end
+ end
+
+ context 'when Hash object is given' do
+ let(:object) { { question: :answer } }
+ let(:enumerable) { items }
+ let(:header_to_value_hash) do
+ { 'Q & A' => :question, 'Reversed' => ->(o) { o[:question].to_s.reverse } }
+ end
+
+ it_behaves_like 'csv builder examples'
+
+ it_behaves_like 'excel sanitization' do
+ let(:dangerous_title) { { title: "=cmd|' /C calc'!A0 title", description: "*safe_desc" } }
+ let(:dangerous_desc) { { title: "*safe_title", description: "=cmd|' /C calc'!A0 desc" } }
+ let(:header_to_value_hash) { { 'Title' => :title, 'Description' => :description } }
+ end
+
+ it_behaves_like 'csv builder with truncation ability' do
+ let(:big_object) { { question: 'Long' * 1024 } }
+ let(:question_value) { big_object[:question] }
+ end
end
end
diff --git a/gems/gem-pg.gitlab-ci.yml b/gems/gem-pg.gitlab-ci.yml
new file mode 100644
index 00000000000..c48e18fa297
--- /dev/null
+++ b/gems/gem-pg.gitlab-ci.yml
@@ -0,0 +1,80 @@
+# The template generates jobs for gems vendored in the main GitLab project
+# under `gem_path_prefix` (defaults to `gems/`).
+#
+# Inputs
+# - `gem_name`: The name of the gem, i.e. if the gem is located at `gems/gitlab-rspec`, `gem_name` should be set to `gitlab-rspec`.
+# - `gem_path_prefix`: The prefix of the gem path, i.e. if the gem is located at `vendor/gems/gitlab-rspec`, `gem_path_prefix` should be set to `vendor/gems/`. Defaults to `gems/`.
+spec:
+ inputs:
+ gem_name:
+ gem_path_prefix:
+ default: "gems/"
+---
+workflow:
+ name: '[$[[inputs.gem_name]] gem] Ruby $RUBY_VERSION pipeline'
+ rules:
+ - when: always
+
+variables:
+ BUNDLE_PATH: "vendor"
+ BUNDLE_FROZEN: "true"
+ GIT_DEPTH: "20"
+ # 'GIT_STRATEGY: clone' optimizes the pack-objects cache hit ratio
+ GIT_STRATEGY: "clone"
+ GIT_SUBMODULE_STRATEGY: "none"
+ GET_SOURCES_ATTEMPTS: "3"
+ # Default Ruby version for jobs that don't use .ruby_matrix
+ RUBY_VERSION: "3.0"
+
+default:
+ image: "ruby:${RUBY_VERSION}"
+ cache:
+ key: "$[[inputs.gem_name]]-${RUBY_VERSION}"
+ paths:
+ - "$[[inputs.gem_path_prefix]]$[[inputs.gem_name]]/vendor/ruby"
+ before_script:
+ - cp config/{database.yml.postgresql,database.yml}
+ - "sed -i 's/username: postgres$/username: gitlab/g' config/database.yml"
+ - "sed -i 's/password:\\s*$/password: password/g' config/database.yml"
+ - "sed -i 's/host: localhost$/host: postgres/g' config/database.yml"
+ - cd $[[inputs.gem_path_prefix]]$[[inputs.gem_name]]
+ - ruby -v # Print out ruby version for debugging
+ - bundle_version=$(grep -A 1 "BUNDLED WITH" Gemfile.lock | tail -n 1 | sed -e 's/[[:space:]]//')
+ - gem install bundler --version "$bundle_version" --no-document # Bundler is not installed with the image
+ - bundle config # Show bundler configuration
+ - bundle install --jobs=$(nproc) --retry=3
+
+.ruby_matrix:
+ parallel:
+ matrix:
+ - RUBY_VERSION: ["3.0", "3.1", "3.2"]
+
+.ruby_and_postgres_matrix:
+ services:
+ - name: postgres:${POSTGRES_VERSION}
+ command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"]
+ parallel:
+ matrix:
+ - RUBY_VERSION: ["3.0", "3.1", "3.2"]
+ POSTGRES_VERSION: ["12", "13", "14"]
+
+rubocop:
+ extends: .ruby_matrix
+ rules:
+ - exists: ["$[[inputs.gem_path_prefix]]$[[inputs.gem_name]]/.rubocop.yml"]
+ script:
+ - bundle exec rubocop
+
+rspec:
+ extends: .ruby_and_postgres_matrix
+ variables:
+ POSTGRES_USER: gitlab
+ POSTGRES_PASSWORD: password
+ script:
+ - RAILS_ENV=test bundle exec rspec
+ coverage: '/LOC \((\d+\.\d+%)\) covered.$/'
+ artifacts:
+ expire_in: 31d
+ when: always
+ paths:
+ - coverage/
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb
index 01ea934863f..4c46dff868c 100644
--- a/gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli.rb
@@ -6,6 +6,7 @@ module Gitlab
module Cli
autoload :VERSION, 'gitlab/backup/cli/version'
autoload :Runner, 'gitlab/backup/cli/runner'
+ autoload :Utils, 'gitlab/backup/cli/utils'
Error = Class.new(StandardError)
# Your code goes here...
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/utils.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/utils.rb
new file mode 100644
index 00000000000..84e6c605172
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/utils.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Utils
+ autoload :PgDump, 'gitlab/backup/cli/utils/pg_dump'
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/lib/gitlab/backup/cli/utils/pg_dump.rb b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/utils/pg_dump.rb
new file mode 100644
index 00000000000..6c16ca5cf06
--- /dev/null
+++ b/gems/gitlab-backup-cli/lib/gitlab/backup/cli/utils/pg_dump.rb
@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Backup
+ module Cli
+ module Utils
+ class PgDump
+ # Expose snapshot_id to be used when creating a database dump
+ # See https://www.postgresql.org/docs/14/functions-admin.html#FUNCTIONS-SNAPSHOT-SYNCHRONIZATION
+ attr_reader :snapshot_id
+ # Dump only specified database schemas instead of everything
+ attr_reader :schemas
+ # Database name
+ attr_reader :database_name
+ # Additional ENV variables to use when running PgDump
+ attr_reader :env
+
+ # @param [String] database_name
+ # @param [String] snapshot_id the snapshot id to use when creating a database dump
+ # @param [Array<String>] schemas
+ # @param [Hash<String,String>] env
+ def initialize(database_name:, snapshot_id: nil, schemas: [], env: {})
+ @database_name = database_name
+ @snapshot_id = snapshot_id
+ @schemas = schemas
+ @env = env
+ end
+
+ # Spawn a pg_dump process and assign a given output IO
+ #
+ # @param [IO] output the output IO
+ def spawn(output:)
+ Process.spawn(env, 'pg_dump', *cmd_args, out: output)
+ end
+
+ private
+
+ # Returns a list of arguments used by the pg_dump command
+ #
+ # @return [Array<String (frozen)>]
+ def cmd_args
+ args = ["--clean"] # Pass '--clean' to include 'DROP TABLE' statements in the DB dump.
+ args << '--if-exists'
+ args << "--snapshot=#{snapshot_id}" if snapshot_id
+
+ schemas.each do |schema|
+ args << '-n'
+ args << schema
+ end
+
+ args << database_name
+
+ args
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/sig/gitlab/backup/cli.rbs b/gems/gitlab-backup-cli/sig/gitlab/backup/cli.rbs
index 25540c06400..76b68239e30 100644
--- a/gems/gitlab-backup-cli/sig/gitlab/backup/cli.rbs
+++ b/gems/gitlab-backup-cli/sig/gitlab/backup/cli.rbs
@@ -1,6 +1,7 @@
module Gitlab
module Backup
module Cli
+ Error: StandardError
VERSION: String
end
end
diff --git a/gems/gitlab-backup-cli/sig/gitlab/backup/cli/utils/pg_dump.rbs b/gems/gitlab-backup-cli/sig/gitlab/backup/cli/utils/pg_dump.rbs
new file mode 100644
index 00000000000..1718d0df6c0
--- /dev/null
+++ b/gems/gitlab-backup-cli/sig/gitlab/backup/cli/utils/pg_dump.rbs
@@ -0,0 +1,25 @@
+module Gitlab
+ module Backup
+ module Cli
+ module Utils
+ class PgDump
+ attr_reader snapshot_id: String
+ attr_reader schemas: Array[String]
+ attr_reader database_name: String
+ attr_reader env: Hash[String,String]
+
+ def initialize: (database_name: String, ?snapshot_id: String?, ?schemas: Array[String], ?env: Hash[String, String]) -> void
+
+ # Spawn a pg_dump process and assign a given output IO
+ #
+ # @param [IO] output the output IO
+ def spawn: (output: IO) -> Integer
+
+ private
+
+ def cmd_args: () -> Array[String]
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-backup-cli/spec/gitlab/backup/cli/utils/pg_dump_spec.rb b/gems/gitlab-backup-cli/spec/gitlab/backup/cli/utils/pg_dump_spec.rb
new file mode 100644
index 00000000000..b1e8495c637
--- /dev/null
+++ b/gems/gitlab-backup-cli/spec/gitlab/backup/cli/utils/pg_dump_spec.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Backup::Cli::Utils::PgDump do
+ let(:cmd_args) { pg_dump.send(:cmd_args) }
+ let(:database_name) { 'gitlab_database' }
+ let(:env) do
+ {
+ 'PGHOST' => '192.168.99.99',
+ 'PGPORT' => '5434'
+ }
+ end
+
+ subject(:pg_dump) { described_class.new(database_name: database_name) }
+
+ context 'with accessors' do
+ it { respond_to :database_name }
+ it { respond_to :snapshot_id }
+ it { respond_to :schemas }
+ it { respond_to :env }
+ end
+
+ describe '#cmd_args' do
+ let(:default_args) { %w[--clean --if-exists] }
+
+ context 'when no optional parameter is provided' do
+ it 'returns default arguments' do
+ expect(cmd_args).to eq(default_args << database_name)
+ end
+ end
+
+ context 'with custom snapshot_id' do
+ let(:snapshot_id) { '00000003-000001BF-1' }
+
+ subject(:pg_dump) { described_class.new(database_name: database_name, snapshot_id: snapshot_id) }
+
+ it 'adds a flag between default_args and the database name' do
+ expect(cmd_args).to eq(default_args + %W[--snapshot=#{snapshot_id} #{database_name}])
+ end
+ end
+
+ context 'with custom schemas' do
+ let(:schemas) { %w[public gitlab_partitions_dynamic gitlab_partitions_static] }
+
+ subject(:pg_dump) { described_class.new(database_name: database_name, schemas: schemas) }
+
+ it 'adds additional flags for each schema' do
+ schemas_args = %W[-n #{schemas[0]} -n #{schemas[1]} -n #{schemas[2]}]
+ expected_args = (default_args + schemas_args) << database_name
+
+ expect(cmd_args).to eq(expected_args)
+ end
+ end
+ end
+
+ describe '#spawn' do
+ it 'returns a spawned process' do
+ process = instance_double(Process)
+ expect(Process).to receive(:spawn).and_return(process)
+
+ expect(pg_dump.spawn(output: StringIO)).to eq(process)
+ end
+
+ it 'forwards cmd_args to Process spawn' do
+ expect(Process).to receive(:spawn).with({}, 'pg_dump', *cmd_args, any_args)
+
+ pg_dump.spawn(output: StringIO)
+ end
+
+ context 'when env variables are provided' do
+ subject(:pg_dump) { described_class.new(database_name: database_name, env: env) }
+
+ it 'forwards provided env variables to Process spawn' do
+ expect(Process).to receive(:spawn).with(env, 'pg_dump', any_args)
+
+ pg_dump.spawn(output: StringIO)
+ end
+ end
+ end
+end
diff --git a/gems/rspec_flaky/.gitignore b/gems/gitlab-database-load_balancing/.gitignore
index b04a8c840df..b04a8c840df 100644
--- a/gems/rspec_flaky/.gitignore
+++ b/gems/gitlab-database-load_balancing/.gitignore
diff --git a/gems/gitlab-database-load_balancing/.gitlab-ci.yml b/gems/gitlab-database-load_balancing/.gitlab-ci.yml
new file mode 100644
index 00000000000..6816d641291
--- /dev/null
+++ b/gems/gitlab-database-load_balancing/.gitlab-ci.yml
@@ -0,0 +1,4 @@
+include:
+ - local: gems/gem-pg.gitlab-ci.yml
+ inputs:
+ gem_name: "gitlab-database-load_balancing"
diff --git a/gems/rspec_flaky/.rspec b/gems/gitlab-database-load_balancing/.rspec
index 34c5164d9b5..34c5164d9b5 100644
--- a/gems/rspec_flaky/.rspec
+++ b/gems/gitlab-database-load_balancing/.rspec
diff --git a/gems/gitlab-database-load_balancing/.rubocop.yml b/gems/gitlab-database-load_balancing/.rubocop.yml
new file mode 100644
index 00000000000..583fa8227ee
--- /dev/null
+++ b/gems/gitlab-database-load_balancing/.rubocop.yml
@@ -0,0 +1,6 @@
+inherit_from:
+ - ../config/rubocop.yml
+
+Gemfile/MissingFeatureCategory:
+ Exclude:
+ - 'Gemfile'
diff --git a/gems/gitlab-database-load_balancing/Gemfile b/gems/gitlab-database-load_balancing/Gemfile
new file mode 100644
index 00000000000..05b508b1333
--- /dev/null
+++ b/gems/gitlab-database-load_balancing/Gemfile
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in gitlab-safe_request_store.gemspec
+gemspec
+
+group :development, :test do
+ gem 'gitlab-rspec', path: '../gitlab-rspec'
+end
+
+gem 'activerecord-gitlab', path: '../activerecord-gitlab'
+gem 'gitlab-utils', path: '../gitlab-utils'
+gem 'gitlab-safe_request_store', path: '../gitlab-safe_request_store'
diff --git a/gems/gitlab-database-load_balancing/Gemfile.lock b/gems/gitlab-database-load_balancing/Gemfile.lock
new file mode 100644
index 00000000000..b2d66b9a386
--- /dev/null
+++ b/gems/gitlab-database-load_balancing/Gemfile.lock
@@ -0,0 +1,303 @@
+PATH
+ remote: ../activerecord-gitlab
+ specs:
+ activerecord-gitlab (0.2.0)
+ activerecord (>= 7)
+
+PATH
+ remote: ../gitlab-rspec
+ specs:
+ gitlab-rspec (0.1.0)
+ activerecord (>= 6.1, < 8)
+ activesupport (>= 6.1, < 8)
+ rspec (~> 3.0)
+
+PATH
+ remote: ../gitlab-safe_request_store
+ specs:
+ gitlab-safe_request_store (0.1.0)
+ request_store
+
+PATH
+ remote: ../gitlab-utils
+ specs:
+ gitlab-utils (0.1.0)
+ actionview (>= 6.1.7.2)
+ activesupport (>= 6.1.7.2)
+ addressable (~> 2.8)
+ nokogiri (~> 1.15.2)
+ rake (~> 13.0)
+
+PATH
+ remote: .
+ specs:
+ gitlab-database-load_balancing (0.1.0)
+ gitlab-net-dns (~> 0.9.2)
+ pg (~> 1.5.4)
+ rails (~> 7.0.8)
+
+GEM
+ remote: https://rubygems.org/
+ specs:
+ actioncable (7.0.8)
+ actionpack (= 7.0.8)
+ activesupport (= 7.0.8)
+ nio4r (~> 2.0)
+ websocket-driver (>= 0.6.1)
+ actionmailbox (7.0.8)
+ actionpack (= 7.0.8)
+ activejob (= 7.0.8)
+ activerecord (= 7.0.8)
+ activestorage (= 7.0.8)
+ activesupport (= 7.0.8)
+ mail (>= 2.7.1)
+ net-imap
+ net-pop
+ net-smtp
+ actionmailer (7.0.8)
+ actionpack (= 7.0.8)
+ actionview (= 7.0.8)
+ activejob (= 7.0.8)
+ activesupport (= 7.0.8)
+ mail (~> 2.5, >= 2.5.4)
+ net-imap
+ net-pop
+ net-smtp
+ rails-dom-testing (~> 2.0)
+ actionpack (7.0.8)
+ actionview (= 7.0.8)
+ activesupport (= 7.0.8)
+ rack (~> 2.0, >= 2.2.4)
+ rack-test (>= 0.6.3)
+ rails-dom-testing (~> 2.0)
+ rails-html-sanitizer (~> 1.0, >= 1.2.0)
+ actiontext (7.0.8)
+ actionpack (= 7.0.8)
+ activerecord (= 7.0.8)
+ activestorage (= 7.0.8)
+ activesupport (= 7.0.8)
+ globalid (>= 0.6.0)
+ nokogiri (>= 1.8.5)
+ actionview (7.0.8)
+ activesupport (= 7.0.8)
+ builder (~> 3.1)
+ erubi (~> 1.4)
+ rails-dom-testing (~> 2.0)
+ rails-html-sanitizer (~> 1.1, >= 1.2.0)
+ activejob (7.0.8)
+ activesupport (= 7.0.8)
+ globalid (>= 0.3.6)
+ activemodel (7.0.8)
+ activesupport (= 7.0.8)
+ activerecord (7.0.8)
+ activemodel (= 7.0.8)
+ activesupport (= 7.0.8)
+ activestorage (7.0.8)
+ actionpack (= 7.0.8)
+ activejob (= 7.0.8)
+ activerecord (= 7.0.8)
+ activesupport (= 7.0.8)
+ marcel (~> 1.0)
+ mini_mime (>= 1.1.0)
+ activesupport (7.0.8)
+ concurrent-ruby (~> 1.0, >= 1.0.2)
+ i18n (>= 1.6, < 2)
+ minitest (>= 5.1)
+ tzinfo (~> 2.0)
+ addressable (2.8.5)
+ public_suffix (>= 2.0.2, < 6.0)
+ ast (2.4.2)
+ binding_of_caller (1.0.0)
+ debug_inspector (>= 0.0.1)
+ builder (3.2.4)
+ coderay (1.1.3)
+ concurrent-ruby (1.2.2)
+ crass (1.0.6)
+ date (3.3.3)
+ debug_inspector (1.1.0)
+ diff-lcs (1.5.0)
+ erubi (1.12.0)
+ gitlab-net-dns (0.9.2)
+ gitlab-styles (10.1.0)
+ rubocop (~> 1.50.2)
+ rubocop-graphql (~> 0.18)
+ rubocop-performance (~> 1.15)
+ rubocop-rails (~> 2.17)
+ rubocop-rspec (~> 2.22)
+ globalid (1.2.1)
+ activesupport (>= 6.1)
+ i18n (1.12.0)
+ concurrent-ruby (~> 1.0)
+ json (2.6.3)
+ loofah (2.21.4)
+ crass (~> 1.0.2)
+ nokogiri (>= 1.12.0)
+ mail (2.8.1)
+ mini_mime (>= 0.1.1)
+ net-imap
+ net-pop
+ net-smtp
+ marcel (1.0.2)
+ method_source (1.0.0)
+ mini_mime (1.1.5)
+ mini_portile2 (2.8.5)
+ minitest (5.17.0)
+ net-imap (0.4.4)
+ date
+ net-protocol
+ net-pop (0.1.2)
+ net-protocol
+ net-protocol (0.2.1)
+ timeout
+ net-smtp (0.4.0)
+ net-protocol
+ nio4r (2.5.9)
+ nokogiri (1.15.4)
+ mini_portile2 (~> 2.8.2)
+ racc (~> 1.4)
+ parallel (1.22.1)
+ parser (3.2.2.3)
+ ast (~> 2.4.1)
+ racc
+ pg (1.5.4)
+ proc_to_ast (0.1.0)
+ coderay
+ parser
+ unparser
+ pry (0.14.2)
+ coderay (~> 1.1)
+ method_source (~> 1.0)
+ public_suffix (5.0.3)
+ racc (1.6.2)
+ rack (2.2.8)
+ rack-test (2.1.0)
+ rack (>= 1.3)
+ rails (7.0.8)
+ actioncable (= 7.0.8)
+ actionmailbox (= 7.0.8)
+ actionmailer (= 7.0.8)
+ actionpack (= 7.0.8)
+ actiontext (= 7.0.8)
+ actionview (= 7.0.8)
+ activejob (= 7.0.8)
+ activemodel (= 7.0.8)
+ activerecord (= 7.0.8)
+ activestorage (= 7.0.8)
+ activesupport (= 7.0.8)
+ bundler (>= 1.15.0)
+ railties (= 7.0.8)
+ rails-dom-testing (2.2.0)
+ activesupport (>= 5.0.0)
+ minitest
+ nokogiri (>= 1.6)
+ rails-html-sanitizer (1.6.0)
+ loofah (~> 2.21)
+ nokogiri (~> 1.14)
+ railties (7.0.8)
+ actionpack (= 7.0.8)
+ activesupport (= 7.0.8)
+ method_source
+ rake (>= 12.2)
+ thor (~> 1.0)
+ zeitwerk (~> 2.5)
+ rainbow (3.1.1)
+ rake (13.1.0)
+ regexp_parser (2.7.0)
+ request_store (1.5.1)
+ rack (>= 1.4)
+ rexml (3.2.5)
+ rspec (3.12.0)
+ rspec-core (~> 3.12.0)
+ rspec-expectations (~> 3.12.0)
+ rspec-mocks (~> 3.12.0)
+ rspec-core (3.12.1)
+ rspec-support (~> 3.12.0)
+ rspec-expectations (3.12.2)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.12.0)
+ rspec-mocks (3.12.3)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.12.0)
+ rspec-parameterized (1.0.0)
+ rspec-parameterized-core (< 2)
+ rspec-parameterized-table_syntax (< 2)
+ rspec-parameterized-core (1.0.0)
+ parser
+ proc_to_ast
+ rspec (>= 2.13, < 4)
+ unparser
+ rspec-parameterized-table_syntax (1.0.1)
+ binding_of_caller
+ rspec-parameterized-core (< 2)
+ rspec-rails (6.0.3)
+ actionpack (>= 6.1)
+ activesupport (>= 6.1)
+ railties (>= 6.1)
+ rspec-core (~> 3.12)
+ rspec-expectations (~> 3.12)
+ rspec-mocks (~> 3.12)
+ rspec-support (~> 3.12)
+ rspec-support (3.12.0)
+ rubocop (1.50.2)
+ json (~> 2.3)
+ parallel (~> 1.10)
+ parser (>= 3.2.0.0)
+ rainbow (>= 2.2.2, < 4.0)
+ regexp_parser (>= 1.8, < 3.0)
+ rexml (>= 3.2.5, < 4.0)
+ rubocop-ast (>= 1.28.0, < 2.0)
+ ruby-progressbar (~> 1.7)
+ unicode-display_width (>= 2.4.0, < 3.0)
+ rubocop-ast (1.29.0)
+ parser (>= 3.2.1.0)
+ rubocop-capybara (2.18.0)
+ rubocop (~> 1.41)
+ rubocop-factory_bot (2.23.1)
+ rubocop (~> 1.33)
+ rubocop-graphql (0.19.0)
+ rubocop (>= 0.87, < 2)
+ rubocop-performance (1.18.0)
+ rubocop (>= 1.7.0, < 2.0)
+ rubocop-ast (>= 0.4.0)
+ rubocop-rails (2.20.2)
+ activesupport (>= 4.2.0)
+ rack (>= 1.1)
+ rubocop (>= 1.33.0, < 2.0)
+ rubocop-rspec (2.22.0)
+ rubocop (~> 1.33)
+ rubocop-capybara (~> 2.17)
+ rubocop-factory_bot (~> 2.22)
+ ruby-progressbar (1.11.0)
+ thor (1.3.0)
+ timeout (0.4.0)
+ tzinfo (2.0.6)
+ concurrent-ruby (~> 1.0)
+ unicode-display_width (2.4.2)
+ unparser (0.6.8)
+ diff-lcs (~> 1.3)
+ parser (>= 3.2.0)
+ websocket-driver (0.7.6)
+ websocket-extensions (>= 0.1.0)
+ websocket-extensions (0.1.5)
+ zeitwerk (2.6.12)
+
+PLATFORMS
+ ruby
+
+DEPENDENCIES
+ activerecord-gitlab!
+ gitlab-database-load_balancing!
+ gitlab-rspec!
+ gitlab-safe_request_store!
+ gitlab-styles (~> 10.1.0)
+ gitlab-utils!
+ pg (~> 1.5.4)
+ pry
+ rspec (~> 3.0)
+ rspec-parameterized (~> 1.0)
+ rspec-rails (~> 6.0.1)
+ rubocop (~> 1.50)
+ rubocop-rspec (~> 2.22)
+
+BUNDLED WITH
+ 2.4.16
diff --git a/gems/gitlab-database-load_balancing/README.md b/gems/gitlab-database-load_balancing/README.md
new file mode 100644
index 00000000000..e1a963dba50
--- /dev/null
+++ b/gems/gitlab-database-load_balancing/README.md
@@ -0,0 +1,3 @@
+# GitLab Database Load Balancing
+
+This gem is a stub for a move of all `Gitlab::Database::LoadBalancing` code.
diff --git a/gems/gitlab-database-load_balancing/Rakefile b/gems/gitlab-database-load_balancing/Rakefile
new file mode 100644
index 00000000000..cca71754493
--- /dev/null
+++ b/gems/gitlab-database-load_balancing/Rakefile
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]
diff --git a/gems/gitlab-database-load_balancing/gitlab-database-load_balancing.gemspec b/gems/gitlab-database-load_balancing/gitlab-database-load_balancing.gemspec
new file mode 100644
index 00000000000..3abf196ed20
--- /dev/null
+++ b/gems/gitlab-database-load_balancing/gitlab-database-load_balancing.gemspec
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+Gem::Specification.new do |spec|
+ spec.name = "gitlab-database-load_balancing"
+ spec.version = "0.1.0"
+ spec.authors = ["group::database"]
+ spec.email = ["engineering@gitlab.com"]
+
+ spec.summary = "GitLab specific support for read-only replicas"
+ spec.description = "Provides a code on top of existing databases to support read-only replicas."
+ spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-safe_request_store"
+ spec.license = 'MIT'
+ spec.required_ruby_version = ">= 3.0"
+ spec.metadata["rubygems_mfa_required"] = "true"
+
+ spec.files = Dir['lib/**/*.rb']
+ spec.require_paths = ["lib"]
+
+ spec.add_runtime_dependency 'gitlab-net-dns', '~> 0.9.2'
+ spec.add_runtime_dependency "pg", '~> 1.5.4'
+ spec.add_runtime_dependency 'rails', '~> 7.0.8'
+
+ spec.add_development_dependency "gitlab-styles", "~> 10.1.0"
+ spec.add_development_dependency "pg", '~> 1.5.4'
+ spec.add_development_dependency "pry"
+ spec.add_development_dependency "rspec", "~> 3.0"
+ spec.add_development_dependency "rspec-parameterized", "~> 1.0"
+ spec.add_development_dependency "rspec-rails", "~> 6.0.1"
+ spec.add_development_dependency "rubocop", "~> 1.50"
+ spec.add_development_dependency "rubocop-rspec", "~> 2.22"
+end
diff --git a/gems/gitlab-database-load_balancing/spec/spec_helper.rb b/gems/gitlab-database-load_balancing/spec/spec_helper.rb
new file mode 100644
index 00000000000..71faf49cce0
--- /dev/null
+++ b/gems/gitlab-database-load_balancing/spec/spec_helper.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'rails'
+require 'rspec/mocks'
+require 'rspec-parameterized'
+
+require 'gitlab/rspec/all'
+require 'gitlab/utils/all'
+require 'gitlab/safe_request_store'
+
+RSpec.configure do |config|
+ include StubRails
+ include NextInstanceOf
+
+ # Enable flags like --only-failures and --next-failure
+ config.example_status_persistence_file_path = ".rspec_status"
+
+ # Disable RSpec exposing methods globally on `Module` and `main`
+ config.disable_monkey_patching!
+
+ config.expect_with :rspec do |c|
+ c.syntax = :expect
+ end
+
+ config.around(:example, :request_store) do |example|
+ ::Gitlab::SafeRequestStore.ensure_request_store { example.run }
+ end
+end
diff --git a/gems/gitlab-database-lock_retries/.gitignore b/gems/gitlab-database-lock_retries/.gitignore
new file mode 100644
index 00000000000..b04a8c840df
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/.gitignore
@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
diff --git a/gems/gitlab-database-lock_retries/.gitlab-ci.yml b/gems/gitlab-database-lock_retries/.gitlab-ci.yml
new file mode 100644
index 00000000000..afe0b651d5d
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/.gitlab-ci.yml
@@ -0,0 +1,4 @@
+include:
+ - local: gems/gem-pg.gitlab-ci.yml
+ inputs:
+ gem_name: "gitlab-database-lock_retries"
diff --git a/gems/gitlab-database-lock_retries/.rspec b/gems/gitlab-database-lock_retries/.rspec
new file mode 100644
index 00000000000..34c5164d9b5
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/.rspec
@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper
diff --git a/gems/gitlab-database-lock_retries/.rubocop.yml b/gems/gitlab-database-lock_retries/.rubocop.yml
new file mode 100644
index 00000000000..8c670b439d3
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/.rubocop.yml
@@ -0,0 +1,2 @@
+inherit_from:
+ - ../config/rubocop.yml
diff --git a/gems/gitlab-database-lock_retries/CHANGELOG.md b/gems/gitlab-database-lock_retries/CHANGELOG.md
new file mode 100644
index 00000000000..d4fe5180c48
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/CHANGELOG.md
@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2023-11-29
+
+- Initial release
diff --git a/gems/gitlab-database-lock_retries/Gemfile b/gems/gitlab-database-lock_retries/Gemfile
new file mode 100644
index 00000000000..c5cc5cf5513
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/Gemfile
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in gitlab-database-lock_retries.gemspec
+gemspec
diff --git a/gems/gitlab-database-lock_retries/Gemfile.lock b/gems/gitlab-database-lock_retries/Gemfile.lock
new file mode 100644
index 00000000000..36618f6f5ae
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/Gemfile.lock
@@ -0,0 +1,105 @@
+PATH
+ remote: .
+ specs:
+ gitlab-database-lock_retries (0.1.0)
+
+GEM
+ remote: https://rubygems.org/
+ specs:
+ activesupport (7.1.2)
+ base64
+ bigdecimal
+ concurrent-ruby (~> 1.0, >= 1.0.2)
+ connection_pool (>= 2.2.5)
+ drb
+ i18n (>= 1.6, < 2)
+ minitest (>= 5.1)
+ mutex_m
+ tzinfo (~> 2.0)
+ ast (2.4.2)
+ base64 (0.2.0)
+ bigdecimal (3.1.4)
+ concurrent-ruby (1.2.2)
+ connection_pool (2.4.1)
+ diff-lcs (1.5.0)
+ drb (2.2.0)
+ ruby2_keywords
+ gitlab-styles (10.1.0)
+ rubocop (~> 1.50.2)
+ rubocop-graphql (~> 0.18)
+ rubocop-performance (~> 1.15)
+ rubocop-rails (~> 2.17)
+ rubocop-rspec (~> 2.22)
+ i18n (1.14.1)
+ concurrent-ruby (~> 1.0)
+ json (2.6.3)
+ minitest (5.20.0)
+ mutex_m (0.2.0)
+ parallel (1.23.0)
+ parser (3.2.2.4)
+ ast (~> 2.4.1)
+ racc
+ racc (1.7.3)
+ rack (3.0.8)
+ rainbow (3.1.1)
+ regexp_parser (2.8.2)
+ rexml (3.2.6)
+ rspec (3.12.0)
+ rspec-core (~> 3.12.0)
+ rspec-expectations (~> 3.12.0)
+ rspec-mocks (~> 3.12.0)
+ rspec-core (3.12.2)
+ rspec-support (~> 3.12.0)
+ rspec-expectations (3.12.3)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.12.0)
+ rspec-mocks (3.12.6)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.12.0)
+ rspec-support (3.12.1)
+ rubocop (1.50.2)
+ json (~> 2.3)
+ parallel (~> 1.10)
+ parser (>= 3.2.0.0)
+ rainbow (>= 2.2.2, < 4.0)
+ regexp_parser (>= 1.8, < 3.0)
+ rexml (>= 3.2.5, < 4.0)
+ rubocop-ast (>= 1.28.0, < 2.0)
+ ruby-progressbar (~> 1.7)
+ unicode-display_width (>= 2.4.0, < 3.0)
+ rubocop-ast (1.30.0)
+ parser (>= 3.2.1.0)
+ rubocop-capybara (2.19.0)
+ rubocop (~> 1.41)
+ rubocop-factory_bot (2.24.0)
+ rubocop (~> 1.33)
+ rubocop-graphql (0.19.0)
+ rubocop (>= 0.87, < 2)
+ rubocop-performance (1.19.1)
+ rubocop (>= 1.7.0, < 2.0)
+ rubocop-ast (>= 0.4.0)
+ rubocop-rails (2.22.2)
+ activesupport (>= 4.2.0)
+ rack (>= 1.1)
+ rubocop (>= 1.33.0, < 2.0)
+ rubocop-ast (>= 1.30.0, < 2.0)
+ rubocop-rspec (2.25.0)
+ rubocop (~> 1.40)
+ rubocop-capybara (~> 2.17)
+ rubocop-factory_bot (~> 2.22)
+ ruby-progressbar (1.13.0)
+ ruby2_keywords (0.0.5)
+ tzinfo (2.0.6)
+ concurrent-ruby (~> 1.0)
+ unicode-display_width (2.5.0)
+
+PLATFORMS
+ ruby
+
+DEPENDENCIES
+ gitlab-database-lock_retries!
+ gitlab-styles (~> 10.1.0)
+ rspec (~> 3.0)
+
+BUNDLED WITH
+ 2.4.22
diff --git a/gems/gitlab-database-lock_retries/README.md b/gems/gitlab-database-lock_retries/README.md
new file mode 100644
index 00000000000..249ca12a044
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/README.md
@@ -0,0 +1,29 @@
+# Gitlab::Database::LockRetries
+
+This gem provides a way to automatically execute code that relies on acquiring a database lock in a way designed to minimize impact on a busy production database.
+
+## Installation
+
+TODO: Replace `UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG` with your gem name right after releasing it to RubyGems.org. Please do not do it earlier due to security reasons. Alternatively, replace this section with instructions to install your gem from git if you don't plan to release to RubyGems.org.
+
+Install the gem and add to the application's Gemfile by executing:
+
+ $ bundle add UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+ $ gem install UPDATE_WITH_YOUR_GEM_NAME_PRIOR_TO_RELEASE_TO_RUBYGEMS_ORG
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/gitlab-database-lock_retries.
diff --git a/gems/gitlab-database-lock_retries/Rakefile b/gems/gitlab-database-lock_retries/Rakefile
new file mode 100644
index 00000000000..cca71754493
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/Rakefile
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[spec rubocop]
diff --git a/gems/gitlab-database-lock_retries/gitlab-database-lock_retries.gemspec b/gems/gitlab-database-lock_retries/gitlab-database-lock_retries.gemspec
new file mode 100644
index 00000000000..ee019705c3d
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/gitlab-database-lock_retries.gemspec
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require_relative "lib/gitlab/database/lock_retries/version"
+
+Gem::Specification.new do |spec|
+ spec.name = "gitlab-database-lock_retries"
+ spec.version = Gitlab::Database::LockRetries::VERSION
+ spec.authors = ["group::database"]
+ spec.email = ["engineering@gitlab.com"]
+
+ spec.summary = "Gem summary"
+ spec.description = "A more descriptive text about what the gem is doing."
+ spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-database-lock_retries"
+ spec.license = "MIT"
+ spec.required_ruby_version = ">= 3.0"
+ spec.metadata["rubygems_mfa_required"] = "true"
+
+ spec.files = Dir['lib/**/*.rb']
+ spec.require_paths = ["lib"]
+
+ spec.add_development_dependency "gitlab-styles", "~> 10.1.0"
+ spec.add_development_dependency "rspec", "~> 3.0"
+end
diff --git a/gems/gitlab-database-lock_retries/lib/gitlab/database/lock_retries.rb b/gems/gitlab-database-lock_retries/lib/gitlab/database/lock_retries.rb
new file mode 100644
index 00000000000..ea504ea09d5
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/lib/gitlab/database/lock_retries.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative "lock_retries/version"
+
+module Gitlab
+ module Database
+ module LockRetries
+ # Your code goes here...
+ end
+ end
+end
diff --git a/gems/gitlab-database-lock_retries/lib/gitlab/database/lock_retries/version.rb b/gems/gitlab-database-lock_retries/lib/gitlab/database/lock_retries/version.rb
new file mode 100644
index 00000000000..f290842df4b
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/lib/gitlab/database/lock_retries/version.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module LockRetries
+ VERSION = "0.1.0"
+ end
+ end
+end
diff --git a/gems/gitlab-database-lock_retries/spec/gitlab/database/lock_retries_spec.rb b/gems/gitlab-database-lock_retries/spec/gitlab/database/lock_retries_spec.rb
new file mode 100644
index 00000000000..faac321f881
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/spec/gitlab/database/lock_retries_spec.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+RSpec.describe Gitlab::Database::LockRetries do
+ it "has a version number" do
+ expect(Gitlab::Database::LockRetries::VERSION).not_to be nil
+ end
+
+ xit "does something useful"
+end
diff --git a/gems/gitlab-database-lock_retries/spec/spec_helper.rb b/gems/gitlab-database-lock_retries/spec/spec_helper.rb
new file mode 100644
index 00000000000..95fdf74bf99
--- /dev/null
+++ b/gems/gitlab-database-lock_retries/spec/spec_helper.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "gitlab/database/lock_retries"
+
+RSpec.configure do |config|
+ # Enable flags like --only-failures and --next-failure
+ config.example_status_persistence_file_path = ".rspec_status"
+
+ # Disable RSpec exposing methods globally on `Module` and `main`
+ config.disable_monkey_patching!
+
+ config.expect_with :rspec do |c|
+ c.syntax = :expect
+ end
+end
diff --git a/gems/gitlab-housekeeper/.gitignore b/gems/gitlab-housekeeper/.gitignore
new file mode 100644
index 00000000000..48cc0651718
--- /dev/null
+++ b/gems/gitlab-housekeeper/.gitignore
@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
+/*.gem
diff --git a/gems/rspec_flaky/.gitlab-ci.yml b/gems/gitlab-housekeeper/.gitlab-ci.yml
index 41fac86e7a5..2d7f34238a2 100644
--- a/gems/rspec_flaky/.gitlab-ci.yml
+++ b/gems/gitlab-housekeeper/.gitlab-ci.yml
@@ -1,4 +1,4 @@
include:
- local: gems/gem.gitlab-ci.yml
inputs:
- gem_name: "rspec_flaky"
+ gem_name: "gitlab-housekeeper"
diff --git a/gems/gitlab-housekeeper/.rspec b/gems/gitlab-housekeeper/.rspec
new file mode 100644
index 00000000000..34c5164d9b5
--- /dev/null
+++ b/gems/gitlab-housekeeper/.rspec
@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper
diff --git a/gems/gitlab-housekeeper/.rubocop.yml b/gems/gitlab-housekeeper/.rubocop.yml
new file mode 100644
index 00000000000..ebe21242469
--- /dev/null
+++ b/gems/gitlab-housekeeper/.rubocop.yml
@@ -0,0 +1,14 @@
+inherit_from:
+ - ../config/rubocop.yml
+
+Gitlab/HTTParty:
+ Enabled: false
+
+Rails/Output:
+ Enabled: false
+
+Gemfile/MissingFeatureCategory:
+ Enabled: false
+
+Rails/NegateInclude:
+ Enabled: false
diff --git a/gems/gitlab-housekeeper/Gemfile b/gems/gitlab-housekeeper/Gemfile
new file mode 100644
index 00000000000..88ede059510
--- /dev/null
+++ b/gems/gitlab-housekeeper/Gemfile
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in gitlab-housekeeper.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "rspec", "~> 3.0"
+gem "pry"
+gem 'webmock'
+
+group :development, :test do
+ gem 'gitlab-rspec', path: '../gitlab-rspec'
+end
diff --git a/gems/gitlab-housekeeper/Gemfile.lock b/gems/gitlab-housekeeper/Gemfile.lock
new file mode 100644
index 00000000000..2b18c02558c
--- /dev/null
+++ b/gems/gitlab-housekeeper/Gemfile.lock
@@ -0,0 +1,139 @@
+PATH
+ remote: ../gitlab-rspec
+ specs:
+ gitlab-rspec (0.1.0)
+ activerecord (>= 6.1, < 8)
+ activesupport (>= 6.1, < 8)
+ rspec (~> 3.0)
+
+PATH
+ remote: .
+ specs:
+ gitlab-housekeeper (0.1.0)
+ httparty
+ rubocop
+
+GEM
+ remote: https://rubygems.org/
+ specs:
+ activemodel (7.0.8)
+ activesupport (= 7.0.8)
+ activerecord (7.0.8)
+ activemodel (= 7.0.8)
+ activesupport (= 7.0.8)
+ activesupport (7.0.8)
+ concurrent-ruby (~> 1.0, >= 1.0.2)
+ i18n (>= 1.6, < 2)
+ minitest (>= 5.1)
+ tzinfo (~> 2.0)
+ addressable (2.8.5)
+ public_suffix (>= 2.0.2, < 6.0)
+ ast (2.4.2)
+ coderay (1.1.3)
+ concurrent-ruby (1.2.2)
+ crack (0.4.3)
+ safe_yaml (~> 1.0.0)
+ diff-lcs (1.5.0)
+ gitlab-styles (10.1.0)
+ rubocop (~> 1.50.2)
+ rubocop-graphql (~> 0.18)
+ rubocop-performance (~> 1.15)
+ rubocop-rails (~> 2.17)
+ rubocop-rspec (~> 2.22)
+ hashdiff (1.0.1)
+ httparty (0.21.0)
+ mini_mime (>= 1.0.0)
+ multi_xml (>= 0.5.2)
+ i18n (1.14.1)
+ concurrent-ruby (~> 1.0)
+ json (2.7.1)
+ method_source (1.0.0)
+ mini_mime (1.1.5)
+ minitest (5.20.0)
+ multi_xml (0.6.0)
+ parallel (1.23.0)
+ parser (3.2.2.4)
+ ast (~> 2.4.1)
+ racc
+ pry (0.14.2)
+ coderay (~> 1.1)
+ method_source (~> 1.0)
+ public_suffix (5.0.3)
+ racc (1.7.3)
+ rack (3.0.8)
+ rainbow (3.1.1)
+ rake (13.1.0)
+ regexp_parser (2.8.3)
+ rexml (3.2.6)
+ rspec (3.12.0)
+ rspec-core (~> 3.12.0)
+ rspec-expectations (~> 3.12.0)
+ rspec-mocks (~> 3.12.0)
+ rspec-core (3.12.2)
+ rspec-support (~> 3.12.0)
+ rspec-expectations (3.12.3)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.12.0)
+ rspec-mocks (3.12.6)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.12.0)
+ rspec-rails (1.3.2)
+ rack (>= 1.0.0)
+ rspec (>= 1.3.0)
+ rspec-support (3.12.1)
+ rubocop (1.50.2)
+ json (~> 2.3)
+ parallel (~> 1.10)
+ parser (>= 3.2.0.0)
+ rainbow (>= 2.2.2, < 4.0)
+ regexp_parser (>= 1.8, < 3.0)
+ rexml (>= 3.2.5, < 4.0)
+ rubocop-ast (>= 1.28.0, < 2.0)
+ ruby-progressbar (~> 1.7)
+ unicode-display_width (>= 2.4.0, < 3.0)
+ rubocop-ast (1.30.0)
+ parser (>= 3.2.1.0)
+ rubocop-capybara (2.19.0)
+ rubocop (~> 1.41)
+ rubocop-factory_bot (2.24.0)
+ rubocop (~> 1.33)
+ rubocop-graphql (0.19.0)
+ rubocop (>= 0.87, < 2)
+ rubocop-performance (1.19.1)
+ rubocop (>= 1.7.0, < 2.0)
+ rubocop-ast (>= 0.4.0)
+ rubocop-rails (2.22.2)
+ activesupport (>= 4.2.0)
+ rack (>= 1.1)
+ rubocop (>= 1.33.0, < 2.0)
+ rubocop-ast (>= 1.30.0, < 2.0)
+ rubocop-rspec (2.25.0)
+ rubocop (~> 1.40)
+ rubocop-capybara (~> 2.17)
+ rubocop-factory_bot (~> 2.22)
+ ruby-progressbar (1.13.0)
+ safe_yaml (1.0.4)
+ tzinfo (2.0.6)
+ concurrent-ruby (~> 1.0)
+ unicode-display_width (2.5.0)
+ webmock (3.19.1)
+ addressable (>= 2.8.0)
+ crack (>= 0.3.2)
+ hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+ arm64-darwin-22
+
+DEPENDENCIES
+ gitlab-housekeeper!
+ gitlab-rspec!
+ gitlab-styles
+ pry
+ rake (~> 13.0)
+ rspec (~> 3.0)
+ rspec-rails
+ rubocop-rspec
+ webmock
+
+BUNDLED WITH
+ 2.4.21
diff --git a/gems/gitlab-housekeeper/README.md b/gems/gitlab-housekeeper/README.md
new file mode 100644
index 00000000000..f707b99c6f0
--- /dev/null
+++ b/gems/gitlab-housekeeper/README.md
@@ -0,0 +1,34 @@
+# Gitlab::Housekeeper
+
+Housekeeping following https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134487
+
+## Running
+
+Technically you can skip steps 1-2 below if you don't want to create a fork but
+it's recommended as using a bot account with no permissions in
+`gitlab-org/gitlab` will ensure we can't cause much damage if the script makes
+a mistake. The alternative of using your own API token with it's permissions to
+`gitlab-org/gitlab` has slightly more risks.
+
+1. Create a fork of `gitlab-org/gitlab` where your MRs will come from
+1. Create a project access token for that project
+1. Set `housekeeper` remote to the fork you created
+ ```
+ git remote add housekeeper git@gitlab.com:DylanGriffith/gitlab.git
+ ```
+1. Open a Postgres.ai tunnel on localhost port 6305
+1. Set the Postgres AI env vars matching the tunnel details for your tunnel
+ ```
+ export POSTGRES_AI_CONNECTION_STRING='host=localhost port=6305 user=dylan dbname=gitlabhq_dblab'
+ export POSTGRES_AI_PASSWORD='the-password'
+ ```
+1. Set the GitLab client details. Will be used to create MR from housekeeper remote:
+ ```
+ export HOUSEKEEPER_FORK_PROJECT_ID=52263761 # Same project as housekeeper remote
+ export HOUSEKEEPER_TARGET_PROJECT_ID=52263761 # Can be 278964 (gitlab-org/gitlab) when ready to create real MRs
+ export HOUSEKEEPER_GITLAB_API_TOKEN=the-api-token
+ ```
+1. Run it:
+ ```
+ bundle exec gitlab-housekeeper -d -m3 -r keeps/overdue_finalize_background_migration.rb -k Keeps::OverdueFinalizeBackgroundMigration
+ ```
diff --git a/gems/gitlab-housekeeper/Rakefile b/gems/gitlab-housekeeper/Rakefile
new file mode 100644
index 00000000000..b6ae734104e
--- /dev/null
+++ b/gems/gitlab-housekeeper/Rakefile
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec
diff --git a/gems/gitlab-housekeeper/bin/gitlab-housekeeper b/gems/gitlab-housekeeper/bin/gitlab-housekeeper
new file mode 100755
index 00000000000..75c547f81ff
--- /dev/null
+++ b/gems/gitlab-housekeeper/bin/gitlab-housekeeper
@@ -0,0 +1,32 @@
+#!/usr/bin/env ruby
+
+require "optparse"
+require 'gitlab/housekeeper'
+
+options = {}
+
+OptionParser.new do |opts|
+ opts.banner = 'Creates merge requests that can be inferred from the current state of the codebase'
+
+ opts.on('-m=M', '--max-mrs=M', Integer, 'Limit of MRs to create. Defaults to 1.') do |m|
+ options[:max_mrs] = m
+ end
+
+ opts.on('-d', '--dry-run', 'Dry-run only. Print the MR titles, descriptions and diffs') do
+ options[:dry_run] = true
+ end
+
+ opts.on('-r lib/foo.rb lib/bar.rb', '--require lib/foo.rb lib/bar.rb', Array, 'Require keeps specified') do |r|
+ options[:require] = r
+ end
+
+ opts.on('-k OverdueFinalizeBackgroundMigration,AnotherKeep', '--keeps OverdueFinalizeBackgroundMigration,AnotherKeep', Array, 'Require keeps specified') do |k|
+ options[:keeps] = k
+ end
+
+ opts.on('-h', '--help', 'Prints this help') do
+ abort opts.to_s
+ end
+end.parse!
+
+Gitlab::Housekeeper::Runner.new(**options).run
diff --git a/gems/gitlab-housekeeper/gitlab-housekeeper.gemspec b/gems/gitlab-housekeeper/gitlab-housekeeper.gemspec
new file mode 100644
index 00000000000..798ca5dcfe6
--- /dev/null
+++ b/gems/gitlab-housekeeper/gitlab-housekeeper.gemspec
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative 'lib/gitlab/housekeeper/version'
+
+Gem::Specification.new do |spec|
+ spec.name = "gitlab-housekeeper"
+ spec.version = Gitlab::Housekeeper::VERSION
+ spec.authors = ["group::tenant-scale"]
+ spec.email = ["engineering@gitlab.com"]
+
+ spec.summary = "Gem summary"
+ spec.description = "Housekeeping following https://gitlab.com/gitlab-org/gitlab/-/merge_requests/134487"
+ spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-housekeeper"
+ spec.license = "MIT"
+ spec.required_ruby_version = ">= 3.0"
+ spec.metadata["rubygems_mfa_required"] = "true"
+
+ spec.files = Dir['lib/**/*.rb']
+ spec.require_paths = ["lib"]
+ spec.executables = ['gitlab-housekeeper']
+
+ spec.add_runtime_dependency 'httparty'
+ spec.add_runtime_dependency 'rubocop'
+
+ spec.add_development_dependency 'gitlab-styles'
+ spec.add_development_dependency 'rspec-rails'
+ spec.add_development_dependency "rubocop-rspec"
+ spec.add_development_dependency 'webmock'
+end
diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper.rb
new file mode 100644
index 00000000000..e9f97e629f4
--- /dev/null
+++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require "gitlab/housekeeper/version"
+require "gitlab/housekeeper/runner"
+
+module Gitlab
+ module Housekeeper
+ Error = Class.new(StandardError)
+ # Your code goes here...
+ end
+end
diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/git.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/git.rb
new file mode 100644
index 00000000000..94bddbaf95f
--- /dev/null
+++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/git.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require 'logger'
+require 'gitlab/housekeeper/shell'
+
+module Gitlab
+ module Housekeeper
+ class Git
+ def initialize(logger:, branch_from: 'master')
+ @logger = logger
+ @branch_from = branch_from
+ end
+
+ def commit_in_branch(change)
+ branch_name = branch_name(change.identifiers)
+
+ create_commit(branch_name, change)
+
+ branch_name
+ end
+
+ def with_branch_from_branch
+ stashed = false
+ current_branch = Shell.execute('git', 'branch', '--show-current').chomp
+
+ result = Shell.execute('git', 'stash')
+ stashed = !result.include?('No local changes to save')
+
+ Shell.execute("git", "checkout", @branch_from)
+
+ yield
+ ensure
+ Shell.execute("git", "checkout", current_branch)
+ Shell.execute('git', 'stash', 'pop') if stashed
+ end
+
+ def create_commit(branch_name, change)
+ current_branch = Shell.execute('git', 'branch', '--show-current').chomp
+
+ begin
+ Shell.execute("git", "branch", '-D', branch_name)
+ rescue Shell::Error # Might not exist yet
+ end
+
+ Shell.execute("git", "checkout", "-b", branch_name)
+ Shell.execute("git", "add", *change.changed_files)
+
+ commit_message = <<~MSG
+ #{change.title}
+
+ #{change.description}
+
+ This commit was generated by
+ [gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139492).
+
+ Changelog: other
+ MSG
+
+ Shell.execute("git", "commit", "-m", commit_message)
+
+ ensure
+ Shell.execute("git", "checkout", current_branch)
+ end
+
+ def branch_name(identifiers)
+ # Hyphen-case each identifier then join together with hyphens.
+ branch_name = identifiers
+ .map { |i| i.gsub(/[[:upper:]]/) { |w| "-#{w.downcase}" } }
+ .join('-')
+ .delete_prefix("-")
+
+ # Truncate if it's too long and add a digest
+ if branch_name.length > 240
+ branch_name = branch_name[0...200] + OpenSSL::Digest::SHA256.hexdigest(branch_name)[0...15]
+ end
+
+ branch_name
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/gitlab_client.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/gitlab_client.rb
new file mode 100644
index 00000000000..b28d44195cb
--- /dev/null
+++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/gitlab_client.rb
@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'httparty'
+require 'json'
+
+module Gitlab
+ module Housekeeper
+ class GitlabClient
+ Error = Class.new(StandardError)
+
+ def initialize
+ @token = ENV.fetch("HOUSEKEEPER_GITLAB_API_TOKEN")
+ @base_uri = 'https://gitlab.com/api/v4'
+ end
+
+ def create_or_update_merge_request(
+ source_project_id:,
+ title:,
+ description:,
+ source_branch:,
+ target_branch:,
+ target_project_id:
+ )
+ existing_iid = get_existing_merge_request(
+ source_project_id: source_project_id,
+ source_branch: source_branch,
+ target_branch: target_branch,
+ target_project_id: target_project_id
+ )
+
+ if existing_iid
+ update_existing_merge_request(
+ existing_iid: existing_iid,
+ title: title,
+ description: description,
+ target_project_id: target_project_id
+ )
+ else
+ create_merge_request(
+ source_project_id: source_project_id,
+ title: title,
+ description: description,
+ source_branch: source_branch,
+ target_branch: target_branch,
+ target_project_id: target_project_id
+ )
+ end
+ end
+
+ private
+
+ def get_existing_merge_request(source_project_id:, source_branch:, target_branch:, target_project_id:)
+ response = HTTParty.get("#{@base_uri}/projects/#{target_project_id}/merge_requests",
+ query: {
+ state: :opened,
+ source_branch: source_branch,
+ target_branch: target_branch,
+ source_project_id: source_project_id
+ },
+ headers: {
+ 'Private-Token' => @token
+ })
+
+ unless (200..299).cover?(response.code)
+ raise Error,
+ "Failed with response code: #{response.code} and body:\n#{response.body}"
+ end
+
+ data = JSON.parse(response.body)
+
+ return nil if data.empty?
+
+ iids = data.pluck('iid')
+
+ raise Error, "More than one matching MR exists: iids: #{iids.join(',')}" unless data.size == 1
+
+ iids.first
+ end
+
+ def create_merge_request(
+ source_project_id:, title:, description:, source_branch:, target_branch:,
+ target_project_id:)
+ response = HTTParty.post("#{@base_uri}/projects/#{source_project_id}/merge_requests", body: {
+ title: title,
+ description: description,
+ source_branch: source_branch,
+ target_branch: target_branch,
+ target_project_id: target_project_id
+ }.to_json,
+ headers: {
+ 'Private-Token' => @token,
+ 'Content-Type' => 'application/json'
+ })
+
+ return if (200..299).cover?(response.code)
+
+ raise Error,
+ "Failed with response code: #{response.code} and body:\n#{response.body}"
+ end
+
+ def update_existing_merge_request(existing_iid:, title:, description:, target_project_id:)
+ response = HTTParty.put("#{@base_uri}/projects/#{target_project_id}/merge_requests/#{existing_iid}", body: {
+ title: title,
+ description: description
+ }.to_json,
+ headers: {
+ 'Private-Token' => @token,
+ 'Content-Type' => 'application/json'
+ })
+
+ return if (200..299).cover?(response.code)
+
+ raise Error,
+ "Failed with response code: #{response.code} and body:\n#{response.body}"
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/keep.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/keep.rb
new file mode 100644
index 00000000000..ecd55008a80
--- /dev/null
+++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/keep.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Housekeeper
+ # A Keep is analogous to a Cop in RuboCop. The Keep is responsible for:
+ # - Detecting a specific code change that should be made (eg. removing an old feature flag)
+ # - Making the code change (eg. delete the feature flag YML file)
+ # - Yielding a Change object that describes this change, For example:
+ # ```
+ # yield Gitlab::Housekeeper::Change.new(
+ # identifiers: ['remove-old-ff', 'old_ff_name'], # Unique and stable identifier for branch name
+ # title: "Remove old feature flag old_ff_name as it has been enabled since %15.0",
+ # description: "This feature flag was enabled in 15.0 and is not needed anymore ...",
+ # changed_files: ["config/feature_flags/ops/old_ff_name.yml", "app/models/user.rb"]
+ # )
+ # ```
+ class Keep
+ # The each_change method must update local working copy files and yield a Change object which describes the
+ # specific changed files and other data that will be used to generate a merge request. This is the core
+ # implementation details for a specific housekeeper keep. This does not need to commit the changes or create the
+ # merge request as that is handled by the gitlab-housekeeper gem.
+ #
+ # @yieldparam [Gitlab::Housekeeper::Change]
+ def each_change
+ raise NotImplementedError, "A Keep must implement each_change method"
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb
new file mode 100644
index 00000000000..76d629e29a3
--- /dev/null
+++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/runner.rb
@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require 'gitlab/housekeeper/keep'
+require "gitlab/housekeeper/gitlab_client"
+require "gitlab/housekeeper/git"
+require 'digest'
+
+module Gitlab
+ module Housekeeper
+ Change = Struct.new(:identifiers, :title, :description, :changed_files)
+
+ class Runner
+ def initialize(max_mrs: 1, dry_run: false, require: [], keeps: nil)
+ @max_mrs = max_mrs
+ @dry_run = dry_run
+ @logger = Logger.new($stdout)
+ require_keeps(require)
+
+ @keeps = if keeps
+ keeps.map { |k| k.is_a?(String) ? k.constantize : k }
+ else
+ all_keeps
+ end
+ end
+
+ def run
+ created = 0
+
+ git.with_branch_from_branch do
+ @keeps.each do |keep_class|
+ keep = keep_class.new
+ keep.each_change do |change|
+ branch_name = git.commit_in_branch(change)
+
+ if @dry_run
+ dry_run(change, branch_name)
+ else
+ create(change, branch_name)
+ end
+
+ created += 1
+ break if created >= @max_mrs
+ end
+ break if created >= @max_mrs
+ end
+ end
+
+ puts "Housekeeper created #{created} MRs"
+ end
+
+ def git
+ @git ||= ::Gitlab::Housekeeper::Git.new(logger: @logger)
+ end
+
+ def require_keeps(files)
+ files.each do |r|
+ require(Pathname(r).expand_path.to_s)
+ end
+ end
+
+ def dry_run(change, branch_name)
+ puts
+ puts "# #{change.title}"
+ puts
+ puts change.description
+ puts
+ puts Shell.execute('git', '--no-pager', 'diff', 'master', branch_name, '--', *change.changed_files)
+ end
+
+ def create(change, branch_name)
+ dry_run(change, branch_name)
+
+ Shell.execute('git', 'push', '-f', 'housekeeper', "#{branch_name}:#{branch_name}")
+
+ gitlab_client.create_or_update_merge_request(
+ source_project_id: housekeeper_fork_project_id,
+ title: change.title,
+ description: change.description,
+ source_branch: branch_name,
+ target_branch: 'master',
+ target_project_id: housekeeper_target_project_id
+ )
+ end
+
+ def housekeeper_fork_project_id
+ ENV.fetch('HOUSEKEEPER_FORK_PROJECT_ID')
+ end
+
+ def housekeeper_target_project_id
+ ENV.fetch('HOUSEKEEPER_TARGET_PROJECT_ID')
+ end
+
+ def gitlab_client
+ @gitlab_client ||= GitlabClient.new
+ end
+
+ def all_keeps
+ @all_keeps ||= ObjectSpace.each_object(Class).select { |klass| klass < Keep }
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-housekeeper/lib/gitlab/housekeeper/shell.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/shell.rb
new file mode 100644
index 00000000000..ed51073f0ed
--- /dev/null
+++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/shell.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module Gitlab
+ module Housekeeper
+ class Shell
+ Error = Class.new(StandardError)
+
+ def self.execute(*cmd)
+ stdin, stdout, stderr, wait_thr = Open3.popen3(*cmd)
+
+ stdin.close
+ out = stdout.read
+ stdout.close
+ err = stderr.read
+ stderr.close
+
+ exit_status = wait_thr.value
+
+ raise Error, "Failed with #{exit_status}\n#{out}\n#{err}\n" unless exit_status.success?
+
+ out + err
+ end
+ end
+ end
+end
diff --git a/gems/rspec_flaky/lib/rspec_flaky/version.rb b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/version.rb
index ec507d734c8..da4cce3f5b3 100644
--- a/gems/rspec_flaky/lib/rspec_flaky/version.rb
+++ b/gems/gitlab-housekeeper/lib/gitlab/housekeeper/version.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
-module RspecFlaky
- module Version
+module Gitlab
+ module Housekeeper
VERSION = "0.1.0"
end
end
diff --git a/gems/gitlab-housekeeper/spec/gitlab/housekeeper/git_spec.rb b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/git_spec.rb
new file mode 100644
index 00000000000..8cf69d2b62c
--- /dev/null
+++ b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/git_spec.rb
@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'fileutils'
+require 'tmpdir'
+
+# rubocop:disable RSpec/MultipleMemoizedHelpers
+RSpec.describe ::Gitlab::Housekeeper::Git do
+ let(:logger) { instance_double(Logger, info: nil) }
+ let(:git) { described_class.new(logger: logger) }
+ let(:repository_path) { Pathname(Dir.mktmpdir) }
+ let(:test_branch_name) { 'gitlab-housekeeper--test-branch' }
+ let(:file_in_master) { 'file_in_master.txt' }
+ let(:file_in_another_branch) { 'file_in_another_branch.txt' }
+
+ def setup_master_branch
+ File.write(file_in_master, 'File already in master!')
+
+ ::Gitlab::Housekeeper::Shell.execute('git', 'init')
+ ::Gitlab::Housekeeper::Shell.execute('git', 'checkout', '-b', 'master')
+ ::Gitlab::Housekeeper::Shell.execute('git', 'add', file_in_master)
+ ::Gitlab::Housekeeper::Shell.execute('git', 'commit', '-m', 'Initial commit!')
+ end
+
+ def setup_and_checkout_another_branch
+ ::Gitlab::Housekeeper::Shell.execute('git', 'checkout', '-b', 'another-branch')
+
+ File.write(file_in_another_branch, 'File in another unrelated branch should not be in new branch!')
+ ::Gitlab::Housekeeper::Shell.execute('git', 'add', file_in_another_branch)
+ ::Gitlab::Housekeeper::Shell.execute('git', 'commit', '-m', 'Commit in unrelated branch should not be included')
+ end
+
+ before do
+ @previous_dir = Dir.pwd
+ Dir.chdir(repository_path)
+
+ # Make sure there is a master branch with something to branch from
+ setup_master_branch
+ setup_and_checkout_another_branch
+ end
+
+ after do
+ Dir.chdir(@previous_dir) if @previous_dir # rubocop:disable RSpec/InstanceVariable -- let not suitable for before/after cleanup
+ FileUtils.rm_rf(repository_path)
+ end
+
+ describe '#with_branch_from_branch and #commit_in_branch' do
+ let(:file_not_to_commit) { repository_path.join('test_file_not_to_commit.txt') }
+ let(:test_file1) { 'test_file1.txt' }
+ let(:test_file2) { 'files/test_file2.txt' }
+
+ it 'commits the given change details to the given branch name' do
+ title = "The commit title"
+ description = <<~COMMIT
+ The commit description can be
+ split over multiple lines!
+ COMMIT
+
+ identifiers = %w[GitlabHousekeeper TestBranch]
+
+ Dir.mkdir('files')
+ File.write(test_file1, "Content in file 1!")
+ File.write(test_file2, "Other content in file 2!")
+ File.write(file_not_to_commit, 'Do not commit!')
+
+ changed_files = [test_file1, test_file2]
+
+ change = ::Gitlab::Housekeeper::Change.new(
+ identifiers,
+ title,
+ description,
+ changed_files
+ )
+
+ branch_name = nil
+ git.with_branch_from_branch do
+ branch_name = git.commit_in_branch(change)
+ end
+
+ expect(branch_name).to eq(test_branch_name)
+
+ branches = ::Gitlab::Housekeeper::Shell.execute('git', 'branch')
+ expect(branches).to include(branch_name)
+
+ current_commit_on_another_branch = ::Gitlab::Housekeeper::Shell.execute('git', 'show')
+ expect(current_commit_on_another_branch).to include('Commit in unrelated branch should not be included')
+
+ expected = <<~COMMIT
+ The commit title
+
+ The commit description can be
+ split over multiple lines!
+
+
+ This commit was generated by
+ [gitlab-housekeeper](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/139492).
+
+ Changelog: other
+
+ diff --git a/files/test_file2.txt b/files/test_file2.txt
+ new file mode 100644
+ index 0000000..ff205e0
+ --- /dev/null
+ +++ b/files/test_file2.txt
+ @@ -0,0 +1 @@
+ +Other content in file 2!
+ \\ No newline at end of file
+ diff --git a/test_file1.txt b/test_file1.txt
+ new file mode 100644
+ index 0000000..8dd3371
+ --- /dev/null
+ +++ b/test_file1.txt
+ @@ -0,0 +1 @@
+ +Content in file 1!
+ \\ No newline at end of file
+ COMMIT
+
+ commit = ::Gitlab::Housekeeper::Shell.execute('git', 'show', branch_name).gsub(/\s/, '')
+ expected_without_whitespace = expected.gsub(/\s/, '')
+ expect(commit).to include(expected_without_whitespace)
+
+ ::Gitlab::Housekeeper::Shell.execute('git', 'checkout', branch_name)
+ expect(File).to exist(file_in_master)
+ expect(File).not_to exist(file_in_another_branch)
+ end
+ end
+end
+# rubocop:enable RSpec/MultipleMemoizedHelpers
diff --git a/gems/gitlab-housekeeper/spec/gitlab/housekeeper/gitlab_client_spec.rb b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/gitlab_client_spec.rb
new file mode 100644
index 00000000000..36b2afdc306
--- /dev/null
+++ b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/gitlab_client_spec.rb
@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'gitlab/housekeeper/gitlab_client'
+
+RSpec.describe ::Gitlab::Housekeeper::GitlabClient do
+ let(:client) { described_class.new }
+
+ describe '#create_or_update_merge_request' do
+ let(:params) do
+ {
+ source_project_id: 123,
+ title: 'A new merge request!',
+ description: 'This merge request is pretty good.',
+ source_branch: 'the-source-branch',
+ target_branch: 'the-target-branch',
+ target_project_id: 456
+ }
+ end
+
+ let(:existing_mrs) { [] }
+
+ before do
+ stub_env('HOUSEKEEPER_GITLAB_API_TOKEN', 'the-api-token')
+
+ # Stub the check to see if the merge request already exists
+ stub_request(:get, "https://gitlab.com/api/v4/projects/456/merge_requests?state=opened&source_branch=the-source-branch&target_branch=the-target-branch&source_project_id=123")
+ .with(
+ headers: {
+ 'Private-Token' => 'the-api-token'
+ }
+ )
+ .to_return(status: 200, body: existing_mrs.to_json)
+ end
+
+ it 'calls the GitLab API passing the token' do
+ stub = stub_request(:post, "https://gitlab.com/api/v4/projects/123/merge_requests")
+ .with(
+ body: {
+ title: "A new merge request!",
+ description: "This merge request is pretty good.",
+ source_branch: "the-source-branch",
+ target_branch: "the-target-branch",
+ target_project_id: 456
+ },
+ headers: {
+ 'Content-Type' => 'application/json',
+ 'Private-Token' => 'the-api-token'
+ })
+ .to_return(status: 200, body: "")
+
+ client.create_or_update_merge_request(**params)
+
+ expect(stub).to have_been_requested
+ end
+
+ context 'when the merge request for the branch already exists' do
+ let(:existing_mrs) do
+ [{ iid: 1234 }]
+ end
+
+ it 'updates the merge request' do
+ stub = stub_request(:put, "https://gitlab.com/api/v4/projects/456/merge_requests/1234")
+ .with(
+ body: {
+ title: "A new merge request!",
+ description: "This merge request is pretty good."
+ }.to_json,
+ headers: {
+ 'Content-Type' => 'application/json',
+ 'Private-Token' => 'the-api-token'
+ })
+ .to_return(status: 200, body: "")
+
+ client.create_or_update_merge_request(**params)
+ expect(stub).to have_been_requested
+ end
+
+ context 'when multiple merge requests exist' do
+ let(:existing_mrs) do
+ [{ iid: 1234 }, { iid: 5678 }]
+ end
+
+ it 'raises since we do not expect this to be possible' do
+ expect { client.create_or_update_merge_request(**params) }.to raise_error(described_class::Error)
+ end
+ end
+ end
+
+ it 'raises an error when unsuccessful response' do
+ stub_request(:post, "https://gitlab.com/api/v4/projects/123/merge_requests")
+ .to_return(status: 400, body: "Real bad error")
+
+ expect do
+ client.create_or_update_merge_request(**params)
+ end.to raise_error(described_class::Error, a_string_matching('Real bad error'))
+ end
+ end
+end
diff --git a/gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb
new file mode 100644
index 00000000000..49b4926dbdd
--- /dev/null
+++ b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/runner_spec.rb
@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'gitlab/housekeeper/runner'
+
+RSpec.describe ::Gitlab::Housekeeper::Runner do
+ let(:fake_keep) { instance_double(Class) }
+
+ let(:change1) do
+ ::Gitlab::Housekeeper::Change.new(
+ %w[the identifier for the first change],
+ "The title of MR1",
+ "The description of the MR",
+ ['change1.txt', 'change2.txt']
+ )
+ end
+
+ let(:change2) do
+ ::Gitlab::Housekeeper::Change.new(
+ %w[the identifier for the second change],
+ "The title of MR2",
+ "The description of the MR",
+ ['change1.txt', 'change2.txt']
+ )
+ end
+
+ let(:change3) do
+ ::Gitlab::Housekeeper::Change.new(
+ %w[the identifier for the third change],
+ "The title of MR3",
+ "The description of the MR",
+ ['change1.txt', 'change2.txt']
+ )
+ end
+
+ before do
+ fake_keep_instance = instance_double(::Gitlab::Housekeeper::Keep)
+ allow(fake_keep).to receive(:new).and_return(fake_keep_instance)
+
+ allow(fake_keep_instance).to receive(:each_change)
+ .and_yield(change1)
+ .and_yield(change2)
+ .and_yield(change3)
+ end
+
+ describe '#run' do
+ before do
+ stub_env('HOUSEKEEPER_FORK_PROJECT_ID', '123')
+ stub_env('HOUSEKEEPER_TARGET_PROJECT_ID', '456')
+ end
+
+ it 'loops over the keeps and creates MRs limited by max_mrs' do
+ # Branches get created
+ git = instance_double(::Gitlab::Housekeeper::Git)
+ expect(::Gitlab::Housekeeper::Git).to receive(:new)
+ .and_return(git)
+ expect(git).to receive(:with_branch_from_branch)
+ .and_yield
+ expect(git).to receive(:commit_in_branch).with(change1)
+ .and_return('the-identifier-for-the-first-change')
+ expect(git).to receive(:commit_in_branch).with(change2)
+ .and_return('the-identifier-for-the-second-change')
+
+ # Branches get shown and pushed
+ expect(::Gitlab::Housekeeper::Shell).to receive(:execute)
+ .with('git', '--no-pager', 'diff', 'master',
+ 'the-identifier-for-the-first-change', '--', 'change1.txt', 'change2.txt')
+ expect(::Gitlab::Housekeeper::Shell).to receive(:execute)
+ .with('git', 'push', '-f', 'housekeeper',
+ 'the-identifier-for-the-first-change:the-identifier-for-the-first-change')
+ expect(::Gitlab::Housekeeper::Shell).to receive(:execute)
+ .with('git', '--no-pager', 'diff', 'master',
+ 'the-identifier-for-the-second-change', '--', 'change1.txt', 'change2.txt')
+ expect(::Gitlab::Housekeeper::Shell).to receive(:execute)
+ .with('git', 'push', '-f', 'housekeeper',
+ 'the-identifier-for-the-second-change:the-identifier-for-the-second-change')
+
+ # Merge requests get created
+ gitlab_client = instance_double(::Gitlab::Housekeeper::GitlabClient)
+ expect(::Gitlab::Housekeeper::GitlabClient).to receive(:new)
+ .and_return(gitlab_client)
+ expect(gitlab_client).to receive(:create_or_update_merge_request)
+ .with(
+ source_project_id: '123',
+ title: 'The title of MR1',
+ description: 'The description of the MR',
+ source_branch: 'the-identifier-for-the-first-change',
+ target_branch: 'master',
+ target_project_id: '456'
+ )
+ expect(gitlab_client).to receive(:create_or_update_merge_request)
+ .with(
+ source_project_id: '123',
+ title: 'The title of MR2',
+ description: 'The description of the MR',
+ source_branch: 'the-identifier-for-the-second-change',
+ target_branch: 'master',
+ target_project_id: '456'
+ )
+
+ described_class.new(max_mrs: 2, keeps: [fake_keep]).run
+ end
+ end
+end
diff --git a/gems/gitlab-housekeeper/spec/gitlab/housekeeper/shell_spec.rb b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/shell_spec.rb
new file mode 100644
index 00000000000..2c7b3fb01c8
--- /dev/null
+++ b/gems/gitlab-housekeeper/spec/gitlab/housekeeper/shell_spec.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'gitlab/housekeeper/shell'
+
+RSpec.describe ::Gitlab::Housekeeper::Shell do
+ describe '.execute' do
+ it 'delegates to popen3 and returns stdout' do
+ expect(Open3).to receive(:popen3).with('echo', 'hello world')
+ .and_call_original
+
+ expect(described_class.execute('echo', 'hello world')).to eq("hello world\n")
+ end
+
+ it 'raises when result is not successful' do
+ expect do
+ described_class.execute('cat', 'definitelynotafile')
+ end.to raise_error(
+ described_class::Error,
+ a_string_matching("cat: definitelynotafile: No such file or directory\n")
+ )
+ end
+ end
+end
diff --git a/gems/gitlab-housekeeper/spec/gitlab/housekeeper_spec.rb b/gems/gitlab-housekeeper/spec/gitlab/housekeeper_spec.rb
new file mode 100644
index 00000000000..798ba7f9386
--- /dev/null
+++ b/gems/gitlab-housekeeper/spec/gitlab/housekeeper_spec.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+RSpec.describe Gitlab::Housekeeper do
+ it "has a version number" do
+ expect(Gitlab::Housekeeper::VERSION).not_to be nil
+ end
+end
diff --git a/gems/gitlab-housekeeper/spec/spec_helper.rb b/gems/gitlab-housekeeper/spec/spec_helper.rb
new file mode 100644
index 00000000000..f2ebb2298bd
--- /dev/null
+++ b/gems/gitlab-housekeeper/spec/spec_helper.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require 'rspec/mocks'
+require "gitlab/housekeeper"
+require "gitlab/housekeeper/git"
+require 'webmock/rspec'
+require 'gitlab/rspec/all'
+
+RSpec.configure do |config|
+ config.include StubENV
+ # Enable flags like --only-failures and --next-failure
+ config.example_status_persistence_file_path = ".rspec_status"
+
+ # Disable RSpec exposing methods globally on `Module` and `main`
+ config.disable_monkey_patching!
+
+ config.expect_with :rspec do |c|
+ c.syntax = :expect
+ end
+end
diff --git a/gems/gitlab-http/Gemfile.lock b/gems/gitlab-http/Gemfile.lock
index 1f4910d1d57..5fb1963d8f3 100644
--- a/gems/gitlab-http/Gemfile.lock
+++ b/gems/gitlab-http/Gemfile.lock
@@ -2,6 +2,7 @@ PATH
remote: ../gitlab-rspec
specs:
gitlab-rspec (0.1.0)
+ activerecord (>= 6.1, < 8)
activesupport (>= 6.1, < 8)
rspec (~> 3.0)
@@ -42,6 +43,11 @@ GEM
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.1, >= 1.2.0)
+ activemodel (7.0.7)
+ activesupport (= 7.0.7)
+ activerecord (7.0.7)
+ activemodel (= 7.0.7)
+ activesupport (= 7.0.7)
activesupport (7.0.7)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 1.6, < 2)
@@ -76,11 +82,9 @@ GEM
nokogiri (>= 1.12.0)
method_source (1.0.0)
mini_mime (1.1.2)
- mini_portile2 (2.8.4)
minitest (5.18.1)
multi_xml (0.6.0)
nokogiri (1.15.4)
- mini_portile2 (~> 2.8.2)
racc (~> 1.4)
parallel (1.23.0)
parser (3.2.2.3)
diff --git a/gems/gitlab-http/lib/gitlab/http_v2/client.rb b/gems/gitlab-http/lib/gitlab/http_v2/client.rb
index 52c9ab897f5..147517de706 100644
--- a/gems/gitlab-http/lib/gitlab/http_v2/client.rb
+++ b/gems/gitlab-http/lib/gitlab/http_v2/client.rb
@@ -3,6 +3,7 @@
require 'httparty'
require 'net/http'
require 'active_support/all'
+require 'gitlab/utils/all'
require_relative 'new_connection_adapter'
require_relative 'exceptions'
require_relative 'lazy_response'
@@ -73,13 +74,15 @@ module Gitlab
read_total_timeout = options.fetch(:timeout, DEFAULT_READ_TOTAL_TIMEOUT)
promise = Concurrent::Promise.new do
- httparty_perform_request(http_method, path, options_with_timeouts) do |fragment|
- start_time ||= system_monotonic_time
- elapsed = system_monotonic_time - start_time
+ Gitlab::Utils.restrict_within_concurrent_ruby do
+ httparty_perform_request(http_method, path, options_with_timeouts) do |fragment|
+ start_time ||= system_monotonic_time
+ elapsed = system_monotonic_time - start_time
- raise ReadTotalTimeout, "Request timed out after #{elapsed} seconds" if elapsed > read_total_timeout
+ raise ReadTotalTimeout, "Request timed out after #{elapsed} seconds" if elapsed > read_total_timeout
- yield fragment if block
+ yield fragment if block
+ end
end
end
diff --git a/gems/gitlab-http/lib/gitlab/http_v2/url_blocker.rb b/gems/gitlab-http/lib/gitlab/http_v2/url_blocker.rb
index 878daf42d8a..99876c77953 100644
--- a/gems/gitlab-http/lib/gitlab/http_v2/url_blocker.rb
+++ b/gems/gitlab-http/lib/gitlab/http_v2/url_blocker.rb
@@ -7,6 +7,7 @@ require_relative 'url_allowlist'
module Gitlab
module HTTP_V2
class UrlBlocker
+ GETADDRINFO_TIMEOUT_SECONDS = 15
BlockedUrlError = Class.new(StandardError)
HTTP_PROXY_ENV_VARS = %w[http_proxy https_proxy HTTP_PROXY HTTPS_PROXY].freeze
@@ -192,9 +193,13 @@ module Gitlab
#
# @return [Array<Addrinfo>]
def get_address_info(uri)
- Addrinfo.getaddrinfo(uri.hostname, get_port(uri), nil, :STREAM).map do |addr|
- addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr
+ Timeout.timeout(GETADDRINFO_TIMEOUT_SECONDS) do
+ Addrinfo.getaddrinfo(uri.hostname, get_port(uri), nil, :STREAM).map do |addr|
+ addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr
+ end
end
+ rescue Timeout::Error => e
+ raise Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, e.message
rescue ArgumentError => e
# Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters.
raise unless e.message.include?('hostname too long')
diff --git a/gems/gitlab-http/spec/gitlab/http_v2/url_blocker_spec.rb b/gems/gitlab-http/spec/gitlab/http_v2/url_blocker_spec.rb
index e47098e6f74..904fed9baef 100644
--- a/gems/gitlab-http/spec/gitlab/http_v2/url_blocker_spec.rb
+++ b/gems/gitlab-http/spec/gitlab/http_v2/url_blocker_spec.rb
@@ -214,6 +214,19 @@ RSpec.describe Gitlab::HTTP_V2::UrlBlocker, :stub_invalid_dns_only, feature_cate
end
end
+ context 'when resolving runs into a timeout' do
+ let(:import_url) { 'http://example.com' }
+
+ before do
+ stub_const("#{described_class}::GETADDRINFO_TIMEOUT_SECONDS", 1)
+ allow(Addrinfo).to receive(:getaddrinfo) { sleep 2 }
+ end
+
+ it 'raises an error due to DNS timeout' do
+ expect { subject }.to raise_error(Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError, "execution expired")
+ end
+ end
+
context 'when the URL hostname is a domain' do
context 'when domain can be resolved' do
let(:import_url) { 'https://example.org' }
diff --git a/gems/gitlab-rspec/Gemfile.lock b/gems/gitlab-rspec/Gemfile.lock
index 7dff91cbd2d..d375d858b01 100644
--- a/gems/gitlab-rspec/Gemfile.lock
+++ b/gems/gitlab-rspec/Gemfile.lock
@@ -2,6 +2,7 @@ PATH
remote: .
specs:
gitlab-rspec (0.1.0)
+ activerecord (>= 6.1, < 8)
activesupport (>= 6.1, < 8)
rspec (~> 3.0)
@@ -21,6 +22,11 @@ GEM
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.1, >= 1.2.0)
+ activemodel (7.0.4.3)
+ activesupport (= 7.0.4.3)
+ activerecord (7.0.4.3)
+ activemodel (= 7.0.4.3)
+ activesupport (= 7.0.4.3)
activesupport (7.0.4.3)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 1.6, < 2)
diff --git a/gems/gitlab-rspec/gitlab-rspec.gemspec b/gems/gitlab-rspec/gitlab-rspec.gemspec
index 47c1e420ecd..bfd404df019 100644
--- a/gems/gitlab-rspec/gitlab-rspec.gemspec
+++ b/gems/gitlab-rspec/gitlab-rspec.gemspec
@@ -18,6 +18,7 @@ Gem::Specification.new do |spec|
spec.files = Dir["lib/**/*.rb"]
spec.require_paths = ["lib"]
+ spec.add_runtime_dependency "activerecord", ">= 6.1", "< 8"
spec.add_runtime_dependency "activesupport", ">= 6.1", "< 8"
spec.add_runtime_dependency "rspec", "~> 3.0"
diff --git a/gems/gitlab-rspec/lib/gitlab/rspec/all.rb b/gems/gitlab-rspec/lib/gitlab/rspec/all.rb
index 091d2ba0287..02504a6a881 100644
--- a/gems/gitlab-rspec/lib/gitlab/rspec/all.rb
+++ b/gems/gitlab-rspec/lib/gitlab/rspec/all.rb
@@ -2,6 +2,9 @@
require_relative "../rspec"
require_relative "stub_env"
+require_relative "next_instance_of"
+require_relative "next_found_instance_of"
+require_relative "stub_rails"
require_relative "configurations/time_travel"
diff --git a/gems/gitlab-rspec/lib/gitlab/rspec/next_found_instance_of.rb b/gems/gitlab-rspec/lib/gitlab/rspec/next_found_instance_of.rb
new file mode 100644
index 00000000000..ed402014df4
--- /dev/null
+++ b/gems/gitlab-rspec/lib/gitlab/rspec/next_found_instance_of.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module NextFoundInstanceOf
+ ERROR_MESSAGE = 'NextFoundInstanceOf mock helpers can only be used with ActiveRecord targets'
+ HELPER_METHOD_PATTERN = /(?:allow|expect)_next_found_(?<number>\d+)_instances_of/
+
+ def method_missing(method_name, ...)
+ match_data = method_name.match(HELPER_METHOD_PATTERN)
+ return super unless match_data
+
+ helper_method = method_name.to_s.sub("_#{match_data[:number]}", '')
+
+ public_send(helper_method, *args, match_data[:number].to_i, &block) # rubocop:disable GitlabSecurity/PublicSend -- it is safe
+ end
+
+ def respond_to_missing?(method_name, ...)
+ match_data = method_name.match(HELPER_METHOD_PATTERN)
+ return super unless match_data
+
+ helper_method = method_name.to_s.sub("_#{match_data[:number]}", '')
+ helper_method.respond_to_missing?(helper_method, *args, &block)
+ end
+
+ def expect_next_found_instance_of(klass, &block)
+ expect_next_found_instances_of(klass, nil, &block)
+ end
+
+ def expect_next_found_instances_of(klass, number, &block)
+ check_if_active_record!(klass)
+
+ stub_allocate(expect(klass), klass, number, &block)
+ end
+
+ def allow_next_found_instance_of(klass, &block)
+ allow_next_found_instances_of(klass, nil, &block)
+ end
+
+ def allow_next_found_instances_of(klass, number, &block)
+ check_if_active_record!(klass)
+
+ stub_allocate(allow(klass), klass, number, &block)
+ end
+
+ private
+
+ def check_if_active_record!(klass)
+ raise ArgumentError, ERROR_MESSAGE unless klass < ActiveRecord::Base
+ end
+
+ def stub_allocate(target, klass, number, &_block)
+ stub = receive(:allocate)
+ stub.exactly(number).times if number
+
+ target.to stub.and_wrap_original do |method|
+ method.call.tap do |allocation|
+ # ActiveRecord::Core.allocate returns a frozen object:
+ # https://github.com/rails/rails/blob/291a3d2ef29a3842d1156ada7526f4ee60dd2b59/activerecord/lib/active_record/core.rb#L620
+ # It's unexpected behavior and probably a bug in Rails
+ # Let's work it around by setting the attributes to default to unfreeze the object for now
+ allocation.instance_variable_set(:@attributes, klass._default_attributes)
+
+ yield(allocation)
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-rspec/lib/gitlab/rspec/next_instance_of.rb b/gems/gitlab-rspec/lib/gitlab/rspec/next_instance_of.rb
new file mode 100644
index 00000000000..5cc63fe5c6e
--- /dev/null
+++ b/gems/gitlab-rspec/lib/gitlab/rspec/next_instance_of.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module NextInstanceOf
+ def expect_next_instance_of(klass, *new_args, &blk)
+ stub_new(expect(klass), nil, false, *new_args, &blk)
+ end
+
+ def expect_next_instances_of(klass, number, ordered = false, *new_args, &blk)
+ stub_new(expect(klass), number, ordered, *new_args, &blk)
+ end
+
+ def allow_next_instance_of(klass, *new_args, &blk)
+ stub_new(allow(klass), nil, false, *new_args, &blk)
+ end
+
+ def allow_next_instances_of(klass, number, ordered = false, *new_args, &blk)
+ stub_new(allow(klass), number, ordered, *new_args, &blk)
+ end
+
+ private
+
+ def stub_new(target, number, ordered = false, *new_args, &blk)
+ receive_new = receive(:new)
+ receive_new.ordered if ordered
+ receive_new.with(*new_args) if new_args.present?
+
+ if number.is_a?(Range)
+ receive_new.at_least(number.begin).times if number.begin
+ receive_new.at_most(number.end).times if number.end
+ elsif number
+ receive_new.exactly(number).times
+ end
+
+ target.to receive_new.and_wrap_original do |*original_args, **original_kwargs|
+ method, *original_args = original_args
+ method.call(*original_args, **original_kwargs).tap(&blk)
+ end
+ end
+end
diff --git a/gems/gitlab-rspec/lib/gitlab/rspec/stub_rails.rb b/gems/gitlab-rspec/lib/gitlab/rspec/stub_rails.rb
new file mode 100644
index 00000000000..26eb8151a01
--- /dev/null
+++ b/gems/gitlab-rspec/lib/gitlab/rspec/stub_rails.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_record'
+
+# rubocop:disable Database/MultipleDatabases -- simulate Rails environment
+# rubocop:disable Database/EstablishConnection -- simulate Rails environment
+
+module StubRails
+ extend ActiveSupport::Concern
+
+ included do
+ class RailsApp < Rails::Application # rubocop:disable Lint/ConstantDefinitionInBlock -- load only when included
+ end
+
+ logger = Logger.new($stdout, level: Logger::INFO, formatter: ->(_, _, _, msg) { msg })
+
+ # load timezones
+ begin
+ TZInfo::DataSource.get
+ rescue TZInfo::DataSourceNotFound => e
+ raise e.exception "tzinfo-data is not present. " \
+ "Please add gem 'tzinfo-data' to your Gemfile and run bundle install"
+ end
+ Time.zone_default = Time.find_zone!(Rails.application.config.time_zone)
+
+ ActiveRecord::Base.configurations = Rails.application.config.database_configuration
+
+ # Create and connect to main database
+ begin
+ rails_establish_connection(logger)
+ rescue ActiveRecord::NoDatabaseError
+ rails_create_main_database(logger)
+ rails_establish_connection(logger)
+ end
+ end
+
+ def rails_establish_connection(_logger)
+ ActiveRecord::Base.establish_connection
+ ActiveRecord::Base.connection.execute("SELECT VERSION()")
+ end
+
+ def rails_create_main_database(logger)
+ db_config = ActiveRecord::Base.configurations.find_db_config(Rails.env)
+ logger.info("Creating database #{db_config.database}...")
+
+ ActiveRecord::Base.establish_connection(db_config.configuration_hash.merge(
+ database: "postgres",
+ schema_search_path: "public"
+ ))
+ ActiveRecord::Base.connection.create_database(
+ db_config.database, { encoding: 'utf8' }.merge(db_config.configuration_hash))
+ end
+end
+
+# rubocop:enable Database/MultipleDatabases
+# rubocop:enable Database/EstablishConnection
diff --git a/gems/gitlab-rspec_flaky/.gitignore b/gems/gitlab-rspec_flaky/.gitignore
new file mode 100644
index 00000000000..b04a8c840df
--- /dev/null
+++ b/gems/gitlab-rspec_flaky/.gitignore
@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
diff --git a/gems/gitlab-rspec_flaky/.gitlab-ci.yml b/gems/gitlab-rspec_flaky/.gitlab-ci.yml
new file mode 100644
index 00000000000..926771014ba
--- /dev/null
+++ b/gems/gitlab-rspec_flaky/.gitlab-ci.yml
@@ -0,0 +1,4 @@
+include:
+ - local: gems/gem.gitlab-ci.yml
+ inputs:
+ gem_name: "gitlab-rspec_flaky"
diff --git a/gems/gitlab-rspec_flaky/.rspec b/gems/gitlab-rspec_flaky/.rspec
new file mode 100644
index 00000000000..34c5164d9b5
--- /dev/null
+++ b/gems/gitlab-rspec_flaky/.rspec
@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper
diff --git a/gems/rspec_flaky/.rubocop.yml b/gems/gitlab-rspec_flaky/.rubocop.yml
index 66f8eb6e21c..66f8eb6e21c 100644
--- a/gems/rspec_flaky/.rubocop.yml
+++ b/gems/gitlab-rspec_flaky/.rubocop.yml
diff --git a/gems/rspec_flaky/Gemfile b/gems/gitlab-rspec_flaky/Gemfile
index 90bf29fb647..634ff34a9b3 100644
--- a/gems/rspec_flaky/Gemfile
+++ b/gems/gitlab-rspec_flaky/Gemfile
@@ -2,7 +2,7 @@
source "https://rubygems.org"
-# Specify your gem's dependencies in rspec_flaky.gemspec
+# Specify your gem's dependencies in gitlab-rspec_flaky.gemspec
gemspec
gem "gitlab-rspec", "~> 0.1", path: "../gitlab-rspec"
diff --git a/gems/rspec_flaky/Gemfile.lock b/gems/gitlab-rspec_flaky/Gemfile.lock
index 6be845e81fb..7b5817b91e4 100644
--- a/gems/rspec_flaky/Gemfile.lock
+++ b/gems/gitlab-rspec_flaky/Gemfile.lock
@@ -2,31 +2,48 @@ PATH
remote: ../gitlab-rspec
specs:
gitlab-rspec (0.1.0)
+ activerecord (>= 6.1, < 8)
activesupport (>= 6.1, < 8)
rspec (~> 3.0)
PATH
remote: .
specs:
- rspec_flaky (0.1.0)
+ gitlab-rspec_flaky (0.1.0)
activesupport (>= 6.1, < 8)
rspec (~> 3.0)
GEM
remote: https://rubygems.org/
specs:
- activesupport (7.0.6)
+ activemodel (7.1.1)
+ activesupport (= 7.1.1)
+ activerecord (7.1.1)
+ activemodel (= 7.1.1)
+ activesupport (= 7.1.1)
+ timeout (>= 0.4.0)
+ activesupport (7.1.1)
+ base64
+ bigdecimal
concurrent-ruby (~> 1.0, >= 1.0.2)
+ connection_pool (>= 2.2.5)
+ drb
i18n (>= 1.6, < 2)
minitest (>= 5.1)
+ mutex_m
tzinfo (~> 2.0)
ast (2.4.2)
+ base64 (0.2.0)
+ bigdecimal (3.1.4)
binding_of_caller (1.0.0)
debug_inspector (>= 0.0.1)
coderay (1.1.3)
concurrent-ruby (1.2.2)
+ connection_pool (2.4.1)
debug_inspector (1.1.0)
diff-lcs (1.5.0)
+ drb (2.2.0)
+ ruby2_keywords
gitlab-styles (10.1.0)
rubocop (~> 1.50.2)
rubocop-graphql (~> 0.18)
@@ -37,6 +54,7 @@ GEM
concurrent-ruby (~> 1.0)
json (2.6.3)
minitest (5.18.1)
+ mutex_m (0.2.0)
parallel (1.23.0)
parser (3.2.2.3)
ast (~> 2.4.1)
@@ -104,6 +122,8 @@ GEM
rubocop-capybara (~> 2.17)
rubocop-factory_bot (~> 2.22)
ruby-progressbar (1.13.0)
+ ruby2_keywords (0.0.5)
+ timeout (0.4.1)
tzinfo (2.0.6)
concurrent-ruby (~> 1.0)
unicode-display_width (2.4.2)
@@ -116,9 +136,9 @@ PLATFORMS
DEPENDENCIES
gitlab-rspec (~> 0.1)!
+ gitlab-rspec_flaky!
gitlab-styles (~> 10.1.0)
rspec-parameterized (~> 1.0)
- rspec_flaky!
rubocop (~> 1.50)
rubocop-rspec (~> 2.22)
diff --git a/gems/rspec_flaky/rspec_flaky.gemspec b/gems/gitlab-rspec_flaky/gitlab-rspec_flaky.gemspec
index 6ddbe4afd1e..36bf070d15d 100644
--- a/gems/rspec_flaky/rspec_flaky.gemspec
+++ b/gems/gitlab-rspec_flaky/gitlab-rspec_flaky.gemspec
@@ -1,10 +1,10 @@
# frozen_string_literal: true
-require_relative "lib/rspec_flaky/version"
+require_relative "lib/gitlab/rspec_flaky/version"
Gem::Specification.new do |spec|
- spec.name = "rspec_flaky"
- spec.version = RspecFlaky::Version::VERSION
+ spec.name = "gitlab-rspec_flaky"
+ spec.version = Gitlab::RspecFlaky::Version::VERSION
spec.authors = ["Engineering Productivity"]
spec.email = ["quality@gitlab.com"]
@@ -12,7 +12,7 @@ Gem::Specification.new do |spec|
spec.description =
"This gem provide an RSpec listener that allows to detect flaky examples. See " \
"https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#automatic-retries-and-flaky-tests-detection."
- spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/rspec_flaky"
+ spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-rspec_flaky"
spec.license = "MIT"
spec.required_ruby_version = ">= 3.0"
spec.metadata["rubygems_mfa_required"] = "true"
diff --git a/gems/rspec_flaky/lib/rspec_flaky.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky.rb
index 90fc6b1dc49..4a1a0a0fe10 100644
--- a/gems/rspec_flaky/lib/rspec_flaky.rb
+++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky.rb
@@ -4,3 +4,8 @@ require "rspec"
require_relative "rspec_flaky/config"
require_relative "rspec_flaky/listener"
require_relative "rspec_flaky/version"
+
+module Gitlab
+ module RspecFlaky
+ end
+end
diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/config.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/config.rb
new file mode 100644
index 00000000000..daf5b4f46cc
--- /dev/null
+++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/config.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module RspecFlaky
+ class Config
+ def self.generate_report?
+ !!(ENV['FLAKY_RSPEC_GENERATE_REPORT'] =~ /1|true/)
+ end
+
+ def self.suite_flaky_examples_report_path
+ ENV['FLAKY_RSPEC_SUITE_REPORT_PATH'] || "rspec/flaky/suite-report.json"
+ end
+
+ def self.flaky_examples_report_path
+ ENV['FLAKY_RSPEC_REPORT_PATH'] || "rspec/flaky/report.json"
+ end
+
+ def self.new_flaky_examples_report_path
+ ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || "rspec/flaky/new-report.json"
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/example.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/example.rb
new file mode 100644
index 00000000000..bc3980e1e8d
--- /dev/null
+++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/example.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'forwardable'
+require 'digest'
+
+module Gitlab
+ module RspecFlaky
+ # This is a wrapper class for RSpec::Core::Example
+ class Example
+ extend Forwardable
+
+ def_delegators :execution_result, :status, :exception
+
+ def initialize(rspec_example)
+ @rspec_example = rspec_example.respond_to?(:example) ? rspec_example.example : rspec_example
+ end
+
+ def uid
+ @uid ||= Digest::MD5.hexdigest("#{description}-#{file}") # rubocop:disable Fips/MD5 -- MD5 is only used to compute and ID and we need to keep using for back-compat
+ end
+
+ def example_id
+ rspec_example.id
+ end
+
+ def file
+ metadata[:file_path]
+ end
+
+ def line
+ metadata[:line_number]
+ end
+
+ def description
+ metadata[:full_description]
+ end
+
+ def attempts
+ rspec_example.respond_to?(:attempts) ? rspec_example.attempts : 1
+ end
+
+ def feature_category
+ metadata[:feature_category]
+ end
+
+ def to_h
+ {
+ example_id: example_id,
+ file: file,
+ line: line,
+ description: description,
+ last_attempts_count: attempts,
+ feature_category: feature_category
+ }
+ end
+
+ private
+
+ attr_reader :rspec_example
+
+ def metadata
+ rspec_example.metadata
+ end
+
+ def execution_result
+ rspec_example.execution_result
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_example.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_example.rb
new file mode 100644
index 00000000000..420c6077842
--- /dev/null
+++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_example.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'ostruct'
+
+module Gitlab
+ module RspecFlaky
+ # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file
+ class FlakyExample
+ ALLOWED_ATTRIBUTES = %i[
+ example_id
+ file
+ line
+ description
+ first_flaky_at
+ last_flaky_at
+ last_flaky_job
+ last_attempts_count
+ flaky_reports
+ feature_category
+ ].freeze
+
+ def initialize(example_hash)
+ @attributes = {
+ first_flaky_at: Time.now,
+ last_flaky_at: Time.now,
+ last_flaky_job: nil,
+ last_attempts_count: example_hash[:attempts],
+ flaky_reports: 0,
+ feature_category: example_hash[:feature_category]
+ }.merge(example_hash.slice(*ALLOWED_ATTRIBUTES))
+
+ %i[first_flaky_at last_flaky_at].each do |attr|
+ attributes[attr] = Time.parse(attributes[attr]) if attributes[attr].is_a?(String)
+ end
+ end
+
+ def update!(example_hash)
+ attributes[:file] = example_hash[:file]
+ attributes[:line] = example_hash[:line]
+ attributes[:description] = example_hash[:description]
+ attributes[:first_flaky_at] ||= Time.now
+ attributes[:last_flaky_at] = Time.now
+ attributes[:flaky_reports] += 1
+ attributes[:feature_category] = example_hash[:feature_category]
+ attributes[:last_attempts_count] = example_hash[:last_attempts_count] if example_hash[:last_attempts_count]
+
+ return unless ENV['CI_JOB_URL']
+
+ attributes[:last_flaky_job] = (ENV['CI_JOB_URL']).to_s
+ end
+
+ def to_h
+ attributes.dup
+ end
+
+ private
+
+ attr_reader :attributes
+ end
+ end
+end
diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_examples_collection.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_examples_collection.rb
new file mode 100644
index 00000000000..7250f7bf164
--- /dev/null
+++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_examples_collection.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'active_support/hash_with_indifferent_access'
+require 'delegate'
+
+require_relative 'flaky_example'
+
+module Gitlab
+ module RspecFlaky
+ class FlakyExamplesCollection < SimpleDelegator
+ def initialize(collection = {})
+ raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" unless collection.is_a?(Hash)
+
+ collection_of_flaky_examples =
+ collection.map do |uid, example|
+ [
+ uid,
+ FlakyExample.new(example.to_h.symbolize_keys)
+ ]
+ end
+
+ super(Hash[collection_of_flaky_examples])
+ end
+
+ def to_h
+ transform_values(&:to_h).deep_symbolize_keys
+ end
+
+ def -(other)
+ raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!" unless other.respond_to?(:key)
+
+ self.class.new(reject { |uid, _| other.key?(uid) })
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/listener.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/listener.rb
new file mode 100644
index 00000000000..6f4dce9df33
--- /dev/null
+++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/listener.rb
@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require 'json'
+
+require_relative 'config'
+require_relative 'example'
+require_relative 'flaky_example'
+require_relative 'flaky_examples_collection'
+require_relative 'report'
+
+module Gitlab
+ module RspecFlaky
+ class Listener
+ # - suite_flaky_examples: contains all the currently tracked flacky example
+ # for the whole RSpec suite
+ # - flaky_examples: contains the examples detected as flaky during the
+ # current RSpec run
+ attr_reader :suite_flaky_examples, :flaky_examples
+
+ def initialize(suite_flaky_examples_json = nil)
+ @flaky_examples = FlakyExamplesCollection.new
+ @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json)
+ end
+
+ def example_passed(notification)
+ current_example = Example.new(notification.example)
+
+ return unless current_example.attempts > 1
+
+ flaky_example = suite_flaky_examples.fetch(current_example.uid) do
+ FlakyExample.new(current_example.to_h)
+ end
+ flaky_example.update!(current_example.to_h)
+
+ flaky_examples[current_example.uid] = flaky_example
+ end
+
+ def dump_summary(_)
+ Report.new(flaky_examples).write(Config.flaky_examples_report_path)
+
+ return unless new_flaky_examples.any?
+
+ rails_logger_warn("\nNew flaky examples detected:\n")
+ rails_logger_warn(JSON.pretty_generate(new_flaky_examples.to_h))
+
+ Report.new(new_flaky_examples).write(Config.new_flaky_examples_report_path)
+ end
+
+ private
+
+ def new_flaky_examples
+ @new_flaky_examples ||= flaky_examples - suite_flaky_examples
+ end
+
+ def init_suite_flaky_examples(suite_flaky_examples_json = nil)
+ if suite_flaky_examples_json
+ Report.load_json(suite_flaky_examples_json).flaky_examples
+ else
+ return {} unless File.exist?(Config.suite_flaky_examples_report_path)
+
+ Report.load(Config.suite_flaky_examples_report_path).flaky_examples
+ end
+ end
+
+ def rails_logger_warn(text)
+ target = defined?(Rails) ? Rails.logger : Kernel
+
+ target.warn(text)
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/report.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/report.rb
new file mode 100644
index 00000000000..0fc669d7f0a
--- /dev/null
+++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/report.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'time'
+require 'fileutils'
+
+require_relative 'config'
+require_relative 'flaky_examples_collection'
+
+module Gitlab
+ module RspecFlaky
+ # This class is responsible for loading/saving JSON reports, and pruning
+ # outdated examples.
+ class Report < SimpleDelegator
+ OUTDATED_DAYS_THRESHOLD = 7
+
+ attr_reader :flaky_examples
+
+ def self.load(file_path)
+ load_json(File.read(file_path))
+ end
+
+ def self.load_json(json)
+ new(FlakyExamplesCollection.new(JSON.parse(json)))
+ end
+
+ def initialize(flaky_examples)
+ unless flaky_examples.is_a?(FlakyExamplesCollection)
+ raise ArgumentError,
+ "`flaky_examples` must be a Gitlab::RspecFlaky::FlakyExamplesCollection, #{flaky_examples.class} given!"
+ end
+
+ @flaky_examples = flaky_examples
+ super(flaky_examples)
+ end
+
+ def write(file_path)
+ unless Config.generate_report?
+ Kernel.warn "! Generating reports is disabled. To enable it, please set the `FLAKY_RSPEC_GENERATE_REPORT=1` !"
+ return
+ end
+
+ report_path_dir = File.dirname(file_path)
+ FileUtils.mkdir_p(report_path_dir)
+
+ File.write(file_path, JSON.pretty_generate(flaky_examples.to_h))
+ end
+
+ def prune_outdated(days: OUTDATED_DAYS_THRESHOLD)
+ outdated_date_threshold = Time.now - (3600 * 24 * days)
+ recent_flaky_examples = flaky_examples.dup
+ .delete_if do |_uid, flaky_example|
+ last_flaky_at = flaky_example.to_h[:last_flaky_at]
+ last_flaky_at && last_flaky_at.to_i < outdated_date_threshold.to_i
+ end
+
+ self.class.new(FlakyExamplesCollection.new(recent_flaky_examples))
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/version.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/version.rb
new file mode 100644
index 00000000000..6fae14a1744
--- /dev/null
+++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/version.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module RspecFlaky
+ module Version
+ VERSION = "0.1.0"
+ end
+ end
+end
diff --git a/gems/rspec_flaky/spec/rspec_flaky/config_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/config_spec.rb
index 827249efefa..a3508b123bd 100644
--- a/gems/rspec_flaky/spec/rspec_flaky/config_spec.rb
+++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/config_spec.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
-require 'rspec_flaky/config'
+require 'gitlab/rspec_flaky/config'
-RSpec.describe RspecFlaky::Config, :aggregate_failures do
+RSpec.describe Gitlab::RspecFlaky::Config, :aggregate_failures do
include StubENV
before do
diff --git a/gems/rspec_flaky/spec/rspec_flaky/example_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/example_spec.rb
index 64d65c0e170..8e38b93526a 100644
--- a/gems/rspec_flaky/spec/rspec_flaky/example_spec.rb
+++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/example_spec.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
-require 'rspec_flaky/example'
+require 'gitlab/rspec_flaky/example'
-RSpec.describe RspecFlaky::Example do
+RSpec.describe Gitlab::RspecFlaky::Example do
let(:example_attrs) do
{
id: 'spec/foo/bar_spec.rb:2',
diff --git a/gems/rspec_flaky/spec/rspec_flaky/flaky_example_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/flaky_example_spec.rb
index 244ca275f14..2d4393f2831 100644
--- a/gems/rspec_flaky/spec/rspec_flaky/flaky_example_spec.rb
+++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/flaky_example_spec.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
-require 'rspec_flaky/flaky_example'
+require 'gitlab/rspec_flaky/flaky_example'
-RSpec.describe RspecFlaky::FlakyExample, :aggregate_failures do
+RSpec.describe Gitlab::RspecFlaky::FlakyExample, :aggregate_failures do
include StubENV
let(:example_attrs) do
diff --git a/gems/rspec_flaky/spec/rspec_flaky/flaky_examples_collection_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/flaky_examples_collection_spec.rb
index 260ebc72192..16811ecb2d2 100644
--- a/gems/rspec_flaky/spec/rspec_flaky/flaky_examples_collection_spec.rb
+++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/flaky_examples_collection_spec.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
-require 'rspec_flaky/flaky_examples_collection'
+require 'gitlab/rspec_flaky/flaky_examples_collection'
-RSpec.describe RspecFlaky::FlakyExamplesCollection, :aggregate_failures, :freeze_time do
+RSpec.describe Gitlab::RspecFlaky::FlakyExamplesCollection, :aggregate_failures, :freeze_time do
let(:collection_hash) do
{
a: { example_id: 'spec/foo/bar_spec.rb:2' },
diff --git a/gems/rspec_flaky/spec/rspec_flaky/listener_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/listener_spec.rb
index cbc5422a763..b46044e4521 100644
--- a/gems/rspec_flaky/spec/rspec_flaky/listener_spec.rb
+++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/listener_spec.rb
@@ -1,8 +1,8 @@
# frozen_string_literal: true
-require 'rspec_flaky/listener'
+require 'gitlab/rspec_flaky/listener'
-RSpec.describe RspecFlaky::Listener, :aggregate_failures do
+RSpec.describe Gitlab::RspecFlaky::Listener, :aggregate_failures do
include StubENV
let(:already_flaky_example_uid) { '6e869794f4cfd2badd93eb68719371d1' }
@@ -85,10 +85,10 @@ RSpec.describe RspecFlaky::Listener, :aggregate_failures do
end
it 'delegates the load to RspecFlaky::Report' do
- report = RspecFlaky::Report
- .new(RspecFlaky::FlakyExamplesCollection.new(suite_flaky_example_report))
+ report = Gitlab::RspecFlaky::Report
+ .new(Gitlab::RspecFlaky::FlakyExamplesCollection.new(suite_flaky_example_report))
- expect(RspecFlaky::Report).to receive(:load).with(report_file_path).and_return(report)
+ expect(Gitlab::RspecFlaky::Report).to receive(:load).with(report_file_path).and_return(report)
expect(described_class.new.suite_flaky_examples.to_h).to eq(report.flaky_examples.to_h)
end
end
@@ -99,7 +99,7 @@ RSpec.describe RspecFlaky::Listener, :aggregate_failures do
end
it 'return an empty hash' do
- expect(RspecFlaky::Report).not_to receive(:load)
+ expect(Gitlab::RspecFlaky::Report).not_to receive(:load)
expect(described_class.new.suite_flaky_examples.to_h).to eq({})
end
end
@@ -134,7 +134,7 @@ RSpec.describe RspecFlaky::Listener, :aggregate_failures do
end
it 'changes the flaky examples hash' do
- new_example = RspecFlaky::Example.new(rspec_example)
+ new_example = Gitlab::RspecFlaky::Example.new(rspec_example)
travel_to(Time.now + 42) do
the_future = Time.now
@@ -161,7 +161,7 @@ RSpec.describe RspecFlaky::Listener, :aggregate_failures do
end
it 'changes the all flaky examples hash' do
- new_example = RspecFlaky::Example.new(rspec_example)
+ new_example = Gitlab::RspecFlaky::Example.new(rspec_example)
travel_to(Time.now + 42) do
the_future = Time.now
@@ -215,12 +215,12 @@ RSpec.describe RspecFlaky::Listener, :aggregate_failures do
report1 = double
report2 = double
- expect(RspecFlaky::Report).to receive(:new).with(listener.flaky_examples).and_return(report1)
- expect(report1).to receive(:write).with(RspecFlaky::Config.flaky_examples_report_path)
+ expect(Gitlab::RspecFlaky::Report).to receive(:new).with(listener.flaky_examples).and_return(report1)
+ expect(report1).to receive(:write).with(Gitlab::RspecFlaky::Config.flaky_examples_report_path)
- expect(RspecFlaky::Report)
+ expect(Gitlab::RspecFlaky::Report)
.to receive(:new).with(listener.__send__(:new_flaky_examples)).and_return(report2)
- expect(report2).to receive(:write).with(RspecFlaky::Config.new_flaky_examples_report_path)
+ expect(report2).to receive(:write).with(Gitlab::RspecFlaky::Config.new_flaky_examples_report_path)
listener.dump_summary(nil)
end
diff --git a/gems/rspec_flaky/spec/rspec_flaky/report_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/report_spec.rb
index e1e9bd6a7b1..74d521b1ea1 100644
--- a/gems/rspec_flaky/spec/rspec_flaky/report_spec.rb
+++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/report_spec.rb
@@ -2,9 +2,9 @@
require 'tempfile'
-require 'rspec_flaky/report'
+require 'gitlab/rspec_flaky/report'
-RSpec.describe RspecFlaky::Report, :aggregate_failures, :freeze_time do
+RSpec.describe Gitlab::RspecFlaky::Report, :aggregate_failures, :freeze_time do
let(:thirty_one_days) { 3600 * 24 * 31 }
let(:collection_hash) do
{
@@ -31,7 +31,7 @@ RSpec.describe RspecFlaky::Report, :aggregate_failures, :freeze_time do
}
end
- let(:flaky_examples) { RspecFlaky::FlakyExamplesCollection.new(collection_hash) }
+ let(:flaky_examples) { Gitlab::RspecFlaky::FlakyExamplesCollection.new(collection_hash) }
let(:report) { described_class.new(flaky_examples) }
before do
@@ -67,7 +67,7 @@ RSpec.describe RspecFlaky::Report, :aggregate_failures, :freeze_time do
end
describe '#initialize' do
- it 'accepts a RspecFlaky::FlakyExamplesCollection' do
+ it 'accepts a Gitlab::RspecFlaky::FlakyExamplesCollection' do
expect { report }.not_to raise_error
end
@@ -76,7 +76,7 @@ RSpec.describe RspecFlaky::Report, :aggregate_failures, :freeze_time do
described_class.new([1, 2,
3])
end.to raise_error(ArgumentError,
- "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, Array given!")
+ "`flaky_examples` must be a Gitlab::RspecFlaky::FlakyExamplesCollection, Array given!")
end
end
@@ -95,9 +95,9 @@ RSpec.describe RspecFlaky::Report, :aggregate_failures, :freeze_time do
FileUtils.rm_f(report_file_path)
end
- context 'when RspecFlaky::Config.generate_report? is false' do
+ context 'when Gitlab::RspecFlaky::Config.generate_report? is false' do
before do
- allow(RspecFlaky::Config).to receive(:generate_report?).and_return(false)
+ allow(Gitlab::RspecFlaky::Config).to receive(:generate_report?).and_return(false)
end
it 'does not write any report file' do
@@ -107,12 +107,12 @@ RSpec.describe RspecFlaky::Report, :aggregate_failures, :freeze_time do
end
end
- context 'when RspecFlaky::Config.generate_report? is true' do
+ context 'when Gitlab::RspecFlaky::Config.generate_report? is true' do
before do
- allow(RspecFlaky::Config).to receive(:generate_report?).and_return(true)
+ allow(Gitlab::RspecFlaky::Config).to receive(:generate_report?).and_return(true)
end
- it 'delegates the writes to RspecFlaky::Report' do
+ it 'delegates the writes to Gitlab::RspecFlaky::Report' do
report.write(report_file_path)
expect(File.exist?(report_file_path)).to be(true)
diff --git a/gems/rspec_flaky/spec/spec_helper.rb b/gems/gitlab-rspec_flaky/spec/spec_helper.rb
index 72d48ee6e63..12c60cb17ad 100644
--- a/gems/rspec_flaky/spec/spec_helper.rb
+++ b/gems/gitlab-rspec_flaky/spec/spec_helper.rb
@@ -2,7 +2,7 @@
require "rspec-parameterized"
require "gitlab/rspec/all"
-require "rspec_flaky"
+require "gitlab/rspec_flaky"
RSpec.configure do |config|
# Enable flags like --only-failures and --next-failure
diff --git a/gems/gitlab-secret_detection/.gitignore b/gems/gitlab-secret_detection/.gitignore
new file mode 100644
index 00000000000..99deaff73aa
--- /dev/null
+++ b/gems/gitlab-secret_detection/.gitignore
@@ -0,0 +1,12 @@
+
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
diff --git a/gems/gitlab-secret_detection/.gitlab-ci.yml b/gems/gitlab-secret_detection/.gitlab-ci.yml
new file mode 100644
index 00000000000..53ae91fe4ca
--- /dev/null
+++ b/gems/gitlab-secret_detection/.gitlab-ci.yml
@@ -0,0 +1,4 @@
+include:
+ - local: gems/gem.gitlab-ci.yml
+ inputs:
+ gem_name: "gitlab-secret_detection"
diff --git a/gems/gitlab-secret_detection/.rspec b/gems/gitlab-secret_detection/.rspec
new file mode 100644
index 00000000000..34c5164d9b5
--- /dev/null
+++ b/gems/gitlab-secret_detection/.rspec
@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper
diff --git a/gems/gitlab-secret_detection/.rubocop.yml b/gems/gitlab-secret_detection/.rubocop.yml
new file mode 100644
index 00000000000..1dc800520ca
--- /dev/null
+++ b/gems/gitlab-secret_detection/.rubocop.yml
@@ -0,0 +1,8 @@
+inherit_from:
+ - ../config/rubocop.yml
+
+AllCops:
+ NewCops: enable
+
+RSpec/MultipleMemoizedHelpers:
+ Max: 25
diff --git a/gems/gitlab-secret_detection/CHANGELOG.md b/gems/gitlab-secret_detection/CHANGELOG.md
new file mode 100644
index 00000000000..35648bae90c
--- /dev/null
+++ b/gems/gitlab-secret_detection/CHANGELOG.md
@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0]
+
+- Initial release
diff --git a/gems/gitlab-secret_detection/Gemfile b/gems/gitlab-secret_detection/Gemfile
new file mode 100644
index 00000000000..d0fb4d4d12b
--- /dev/null
+++ b/gems/gitlab-secret_detection/Gemfile
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in gitlab-secret_detection.gemspec
+gemspec
diff --git a/gems/gitlab-secret_detection/Gemfile.lock b/gems/gitlab-secret_detection/Gemfile.lock
new file mode 100644
index 00000000000..dd9f621ee4a
--- /dev/null
+++ b/gems/gitlab-secret_detection/Gemfile.lock
@@ -0,0 +1,149 @@
+PATH
+ remote: .
+ specs:
+ gitlab-secret_detection (0.1.0)
+ re2 (~> 2.4)
+ toml-rb (~> 2.2)
+
+GEM
+ remote: https://rubygems.org/
+ specs:
+ activesupport (7.1.2)
+ base64
+ bigdecimal
+ concurrent-ruby (~> 1.0, >= 1.0.2)
+ connection_pool (>= 2.2.5)
+ drb
+ i18n (>= 1.6, < 2)
+ minitest (>= 5.1)
+ mutex_m
+ tzinfo (~> 2.0)
+ ast (2.4.2)
+ base64 (0.2.0)
+ benchmark-malloc (0.2.0)
+ benchmark-perf (0.6.0)
+ benchmark-trend (0.4.0)
+ bigdecimal (3.1.4)
+ binding_of_caller (1.0.0)
+ debug_inspector (>= 0.0.1)
+ citrus (3.0.2)
+ coderay (1.1.3)
+ concurrent-ruby (1.2.2)
+ connection_pool (2.4.1)
+ debug_inspector (1.1.0)
+ diff-lcs (1.5.0)
+ drb (2.2.0)
+ ruby2_keywords
+ gitlab-styles (11.0.0)
+ rubocop (~> 1.57.1)
+ rubocop-graphql (~> 0.18)
+ rubocop-performance (~> 1.15)
+ rubocop-rails (~> 2.17)
+ rubocop-rspec (~> 2.22)
+ i18n (1.14.1)
+ concurrent-ruby (~> 1.0)
+ json (2.6.3)
+ language_server-protocol (3.17.0.3)
+ mini_portile2 (2.8.5)
+ minitest (5.20.0)
+ mutex_m (0.2.0)
+ parallel (1.23.0)
+ parser (3.2.2.4)
+ ast (~> 2.4.1)
+ racc
+ proc_to_ast (0.1.0)
+ coderay
+ parser
+ unparser
+ racc (1.7.3)
+ rack (3.0.8)
+ rainbow (3.1.1)
+ re2 (2.4.3)
+ mini_portile2 (~> 2.8.5)
+ regexp_parser (2.8.2)
+ rexml (3.2.6)
+ rspec (3.12.0)
+ rspec-core (~> 3.12.0)
+ rspec-expectations (~> 3.12.0)
+ rspec-mocks (~> 3.12.0)
+ rspec-benchmark (0.6.0)
+ benchmark-malloc (~> 0.2)
+ benchmark-perf (~> 0.6)
+ benchmark-trend (~> 0.4)
+ rspec (>= 3.0)
+ rspec-core (3.12.2)
+ rspec-support (~> 3.12.0)
+ rspec-expectations (3.12.3)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.12.0)
+ rspec-mocks (3.12.6)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.12.0)
+ rspec-parameterized (1.0.0)
+ rspec-parameterized-core (< 2)
+ rspec-parameterized-table_syntax (< 2)
+ rspec-parameterized-core (1.0.0)
+ parser
+ proc_to_ast
+ rspec (>= 2.13, < 4)
+ unparser
+ rspec-parameterized-table_syntax (1.0.1)
+ binding_of_caller
+ rspec-parameterized-core (< 2)
+ rspec-support (3.12.1)
+ rubocop (1.57.2)
+ json (~> 2.3)
+ language_server-protocol (>= 3.17.0)
+ parallel (~> 1.10)
+ parser (>= 3.2.2.4)
+ rainbow (>= 2.2.2, < 4.0)
+ regexp_parser (>= 1.8, < 3.0)
+ rexml (>= 3.2.5, < 4.0)
+ rubocop-ast (>= 1.28.1, < 2.0)
+ ruby-progressbar (~> 1.7)
+ unicode-display_width (>= 2.4.0, < 3.0)
+ rubocop-ast (1.30.0)
+ parser (>= 3.2.1.0)
+ rubocop-capybara (2.19.0)
+ rubocop (~> 1.41)
+ rubocop-factory_bot (2.24.0)
+ rubocop (~> 1.33)
+ rubocop-graphql (0.19.0)
+ rubocop (>= 0.87, < 2)
+ rubocop-performance (1.19.1)
+ rubocop (>= 1.7.0, < 2.0)
+ rubocop-ast (>= 0.4.0)
+ rubocop-rails (2.20.0)
+ activesupport (>= 4.2.0)
+ rack (>= 1.1)
+ rubocop (>= 1.33.0, < 2.0)
+ rubocop-rspec (2.25.0)
+ rubocop (~> 1.40)
+ rubocop-capybara (~> 2.17)
+ rubocop-factory_bot (~> 2.22)
+ ruby-progressbar (1.13.0)
+ ruby2_keywords (0.0.5)
+ toml-rb (2.2.0)
+ citrus (~> 3.0, > 3.0)
+ tzinfo (2.0.6)
+ concurrent-ruby (~> 1.0)
+ unicode-display_width (2.5.0)
+ unparser (0.6.10)
+ diff-lcs (~> 1.3)
+ parser (>= 3.2.2.4)
+
+PLATFORMS
+ ruby
+
+DEPENDENCIES
+ gitlab-secret_detection!
+ gitlab-styles (~> 11.0)
+ rspec (~> 3.0)
+ rspec-benchmark (~> 0.6.0)
+ rspec-parameterized (~> 1.0)
+ rubocop (~> 1.57)
+ rubocop-rails (<= 2.20)
+ rubocop-rspec (~> 2.22)
+
+BUNDLED WITH
+ 2.4.22
diff --git a/gems/gitlab-secret_detection/README.md b/gems/gitlab-secret_detection/README.md
new file mode 100644
index 00000000000..1e1095b234a
--- /dev/null
+++ b/gems/gitlab-secret_detection/README.md
@@ -0,0 +1,3 @@
+# Gitlab::SecretDetection
+
+The gitlab-secret_detection gem performs keyword and regex matching on git blobs that may include secrets. The gem accepts one or more git blobs, matches them against a defined ruleset of regular expressions, and returns scan results.
diff --git a/gems/gitlab-secret_detection/gitlab-secret_detection.gemspec b/gems/gitlab-secret_detection/gitlab-secret_detection.gemspec
new file mode 100644
index 00000000000..be9db3aa389
--- /dev/null
+++ b/gems/gitlab-secret_detection/gitlab-secret_detection.gemspec
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "lib/gitlab/secret_detection/version"
+
+Gem::Specification.new do |spec|
+ spec.name = "gitlab-secret_detection"
+ spec.version = Gitlab::SecretDetection::VERSION
+ spec.authors = ["group::static analysis"]
+ spec.email = ["eng-dev-secure-static-analysis@gitlab.com"]
+
+ spec.summary = "The gitlab-secret_detection gem performs regex matching on git blobs that may include secrets."
+ spec.description = "The gitlab-secret_detection gem accepts one or more git blobs,
+ matches them against a defined ruleset of regular expressions (based on gitleaks.toml used by secrets analyzer),
+ and returns scan results."
+ spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-secret_detection"
+ spec.license = "MIT"
+ spec.required_ruby_version = ">= 3.0"
+
+ spec.metadata["rubygems_mfa_required"] = "true"
+ spec.metadata["homepage_uri"] = spec.homepage
+ spec.metadata["source_code_uri"] = spec.homepage
+ spec.metadata["changelog_uri"] = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-secret_detection/CHANGELOG.md"
+
+ spec.files = Dir['lib/**/*.rb']
+ spec.require_paths = ["lib"]
+
+ spec.add_runtime_dependency "re2", "~> 2.4"
+ spec.add_runtime_dependency "toml-rb", "~> 2.2"
+
+ spec.add_development_dependency "gitlab-styles", "~> 11.0"
+ spec.add_development_dependency "rspec", "~> 3.0"
+ spec.add_development_dependency "rspec-benchmark", "~> 0.6.0"
+ spec.add_development_dependency "rspec-parameterized", "~> 1.0"
+ spec.add_development_dependency "rubocop", "~> 1.57"
+ spec.add_development_dependency "rubocop-rails", "<= 2.20" # https://github.com/rubocop/rubocop-rails/issues/1173
+ spec.add_development_dependency "rubocop-rspec", "~> 2.22"
+end
diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection.rb
new file mode 100644
index 00000000000..eadcbc4de43
--- /dev/null
+++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative 'secret_detection/status'
+require_relative 'secret_detection/finding'
+require_relative 'secret_detection/response'
+require_relative 'secret_detection/scan'
+
+module Gitlab
+ module SecretDetection
+ end
+end
diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/finding.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/finding.rb
new file mode 100644
index 00000000000..728d45d6228
--- /dev/null
+++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/finding.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SecretDetection
+ # Finding is a data object representing a secret finding identified within a blob
+ class Finding
+ attr_reader :blob_id, :status, :line_number, :type, :description
+ attr_accessor :occurrences
+
+ def initialize(blob_id, status, line_number = nil, type = nil, description = nil, occurrences = nil) # rubocop:disable Metrics/ParameterLists -- all params are needed
+ @blob_id = blob_id
+ @status = status
+ @line_number = line_number
+ @type = type
+ @description = description
+ @occurrences = occurrences
+ end
+
+ def ==(other)
+ self.class == other.class && other.state == state
+ end
+
+ def to_h
+ {
+ blob_id: blob_id,
+ status: status,
+ line_number: line_number,
+ type: type,
+ description: description,
+ occurrences: occurrences
+ }
+ end
+
+ protected
+
+ def state
+ [blob_id, status, line_number, type, description]
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/response.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/response.rb
new file mode 100644
index 00000000000..a34fba7c0b6
--- /dev/null
+++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/response.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SecretDetection
+ # Response is the data object returned by the scan operation with the following structure
+ #
+ # +status+:: One of values from SecretDetection::Status indicating the scan operation's status
+ # +results+:: Array of SecretDetection::Finding values. Default value is nil.
+ class Response
+ attr_reader :status, :results
+
+ def initialize(status, results = nil)
+ @status = status
+ @results = results
+ end
+
+ def ==(other)
+ self.class == other.class && other.state == state
+ end
+
+ protected
+
+ def state
+ [status, results]
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb
new file mode 100644
index 00000000000..20d630d5dbb
--- /dev/null
+++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb
@@ -0,0 +1,206 @@
+# frozen_string_literal: true
+
+require 'toml-rb'
+require 're2'
+require 'logger'
+require 'timeout'
+
+module Gitlab
+ module SecretDetection
+ # Scan is responsible for running Secret Detection scan operation
+ class Scan
+ # RulesetParseError is thrown when the code fails to parse the
+ # ruleset file from the given path
+ RulesetParseError = Class.new(StandardError)
+
+ # RulesetCompilationError is thrown when the code fails to compile
+ # the predefined rulesets
+ RulesetCompilationError = Class.new(StandardError)
+
+ # default time limit(in seconds) for running the scan operation per invocation
+ DEFAULT_SCAN_TIMEOUT_SECS = 60
+ # default time limit(in seconds) for running the scan operation on a single blob
+ DEFAULT_BLOB_TIMEOUT_SECS = 5
+ # file path where the secrets ruleset file is located
+ RULESET_FILE_PATH = File.expand_path('../../gitleaks.toml', __dir__)
+ # ignore the scanning of a line which ends with the following keyword
+ GITLEAKS_KEYWORD_IGNORE = 'gitleaks:allow'
+
+ # Initializes the instance with logger along with following operations:
+ # 1. Parse ruleset for the given +ruleset_path+(default: +RULESET_FILE_PATH+). Raises +RulesetParseError+
+ # incase the operation fails.
+ # 2. Extract keywords from the parsed ruleset to use it for matching keywords before regex operation.
+ # 3. Build and Compile rule regex patterns obtained from the ruleset. Raises +RulesetCompilationError+
+ # in case the compilation fails.
+ def initialize(logger: Logger.new($stdout), ruleset_path: RULESET_FILE_PATH)
+ @logger = logger
+ @rules = parse_ruleset(ruleset_path)
+ @keywords = create_keywords(rules)
+ @pattern_matcher = build_pattern_matcher(rules)
+ end
+
+ # Runs Secret Detection scan on the list of given blobs. Both the total scan duration and
+ # the duration for each blob is time bound via +timeout+ and +blob_timeout+ respectively.
+ #
+ # +blobs+:: Array of blobs with each blob to have `id` and `data` properties.
+ # +timeout+:: No of seconds(accepts floating point for smaller time values) to limit the total scan duration
+ # +blob_timeout+:: No of seconds(accepts floating point for smaller time values) to limit
+ # the scan duration on each blob
+ #
+ # Returns an instance of SecretDetection::Response by following below structure:
+ # {
+ # status: One of the SecretDetection::Status values
+ # results: [SecretDetection::Finding]
+ # }
+ def secrets_scan(blobs, timeout: DEFAULT_SCAN_TIMEOUT_SECS, blob_timeout: DEFAULT_BLOB_TIMEOUT_SECS)
+ return SecretDetection::Response.new(SecretDetection::Status::INPUT_ERROR) unless validate_scan_input(blobs)
+
+ Timeout.timeout(timeout) do
+ matched_blobs = filter_by_keywords(blobs)
+
+ next SecretDetection::Response.new(SecretDetection::Status::NOT_FOUND) if matched_blobs.empty?
+
+ secrets = find_secrets_bulk(matched_blobs, blob_timeout)
+
+ scan_status = overall_scan_status(secrets)
+
+ SecretDetection::Response.new(scan_status, secrets)
+ end
+ rescue Timeout::Error => e
+ logger.error "Secret detection operation timed out: #{e}"
+
+ SecretDetection::Response.new(SecretDetection::Status::SCAN_TIMEOUT)
+ end
+
+ private
+
+ attr_reader :logger, :rules, :keywords, :pattern_matcher
+
+ # parses given ruleset file and returns the parsed rules
+ def parse_ruleset(ruleset_file_path)
+ rules_data = TomlRB.load_file(ruleset_file_path)
+ rules_data['rules']
+ rescue StandardError => e
+ logger.error "Failed to parse secret detection ruleset from '#{ruleset_file_path}' path: #{e}"
+
+ raise RulesetParseError
+ end
+
+ # builds RE2::Set pattern matcher for the given rules
+ def build_pattern_matcher(rules)
+ matcher = RE2::Set.new
+
+ rules.each do |rule|
+ matcher.add(rule["regex"])
+ end
+
+ unless matcher.compile
+ logger.error "Failed to compile secret detection rulesets in RE::Set"
+
+ raise RulesetCompilationError
+ end
+
+ matcher
+ end
+
+ # creates and returns the unique set of rule matching keywords
+ def create_keywords(rules)
+ secrets_keywords = []
+
+ rules.each do |rule|
+ secrets_keywords << rule["keywords"]
+ end
+
+ secrets_keywords.flatten.compact.to_set
+ end
+
+ # returns only those blobs that contain atleast one of the keywords
+ # from the keywords list
+ def filter_by_keywords(blobs)
+ matched_blobs = []
+
+ blobs.each do |blob|
+ matched_blobs << blob if keywords.any? { |keyword| blob.data.include?(keyword) }
+ end
+
+ matched_blobs.freeze
+ end
+
+ # finds secrets in the given list of blobs
+ def find_secrets_bulk(blobs, blob_timeout)
+ found_secrets = []
+
+ blobs.each do |blob|
+ found_secrets << Timeout.timeout(blob_timeout) { find_secrets(blob) }
+ rescue Timeout::Error => e
+ logger.error "Secret detection scan timed out on the blob(id:#{blob.id}): #{e}"
+
+ found_secrets << SecretDetection::Finding.new(
+ blob.id,
+ SecretDetection::Status::BLOB_TIMEOUT
+ )
+ end
+
+ found_secrets.flatten.freeze
+ end
+
+ # finds secrets in the given blob with a timeout circuit breaker
+ def find_secrets(blob)
+ secrets = []
+
+ blob.data.each_line.with_index do |line, index|
+ # ignore the line scan if it is suffixed with '#gitleaks:allow'
+ next if line.end_with?(GITLEAKS_KEYWORD_IGNORE)
+
+ patterns = pattern_matcher.match(line, :exception => false)
+ next unless patterns.any?
+
+ line_number = index + 1
+ patterns.each do |pattern|
+ type = rules[pattern]["id"]
+ description = rules[pattern]["description"]
+
+ secrets << SecretDetection::Finding.new(
+ blob.id,
+ SecretDetection::Status::FOUND,
+ line_number,
+ type,
+ description
+ )
+ end
+ end
+
+ secrets
+ rescue StandardError => e
+ logger.error "Secret detection scan failed on the blob(id:#{blob.id}): #{e}"
+
+ SecretDetection::Finding.new(blob.id, SecretDetection::Status::SCAN_ERROR)
+ end
+
+ def validate_scan_input(blobs)
+ return false if blobs.nil? || !blobs.instance_of?(Array)
+
+ blobs.all? do |blob|
+ next false unless blob.respond_to?(:id) || blob.respond_to?(:data)
+
+ blob.data.freeze # freeze blobs to avoid additional object allocations on strings
+ end
+ end
+
+ def overall_scan_status(found_secrets)
+ return SecretDetection::Status::NOT_FOUND if found_secrets.empty?
+
+ timed_out_blobs = found_secrets.count { |el| el.status == SecretDetection::Status::BLOB_TIMEOUT }
+
+ case timed_out_blobs
+ when 0
+ SecretDetection::Status::FOUND
+ when found_secrets.length
+ SecretDetection::Status::SCAN_TIMEOUT
+ else
+ SecretDetection::Status::FOUND_WITH_ERRORS
+ end
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb
new file mode 100644
index 00000000000..200294fc2e7
--- /dev/null
+++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SecretDetection
+ # All the possible statuses emitted by the scan operation
+ class Status
+ NOT_FOUND = 0 # When scan operation completes with zero findings
+ FOUND = 1 # When scan operation completes with one or more findings
+ FOUND_WITH_ERRORS = 2 # When scan operation completes with one or more findings along with some errors
+ SCAN_TIMEOUT = 3 # When the scan operation runs beyond given time out
+ BLOB_TIMEOUT = 4 # When the scan operation on a blob runs beyond given time out
+ SCAN_ERROR = 5 # When the scan operation fails due to regex error
+ INPUT_ERROR = 6 # When the scan operation fails due to invalid input
+ end
+ end
+end
diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/version.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/version.rb
new file mode 100644
index 00000000000..8fc73a02121
--- /dev/null
+++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/version.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module SecretDetection
+ VERSION = "0.1.0"
+ end
+end
diff --git a/gems/gitlab-secret_detection/lib/gitleaks.toml b/gems/gitlab-secret_detection/lib/gitleaks.toml
new file mode 100644
index 00000000000..e6c165ff945
--- /dev/null
+++ b/gems/gitlab-secret_detection/lib/gitleaks.toml
@@ -0,0 +1,307 @@
+# This file contains a subset of rules pulled from the original source file.
+# Original Source: https://gitlab.com/gitlab-org/security-products/analyzers/secrets/-/blob/master/gitleaks.toml
+# Reference: https://gitlab.com/gitlab-org/gitlab/-/issues/427011
+title = "gitleaks config"
+
+[[rules]]
+id = "gitlab_personal_access_token"
+description = "GitLab Personal Access Token"
+regex = '''\bglpat-[0-9a-zA-Z_\-]{20}\b'''
+tags = ["gitlab", "revocation_type"]
+keywords = [
+ "glpat",
+]
+
+[[rules]]
+id = "gitlab_pipeline_trigger_token"
+description = "GitLab Pipeline Trigger Token"
+regex = '''\bglptt-[0-9a-zA-Z_\-]{40}\b'''
+tags = ["gitlab"]
+keywords = [
+ "glptt",
+]
+
+[[rules]]
+id = "gitlab_runner_registration_token"
+description = "GitLab Runner Registration Token"
+regex = '''\bGR1348941[0-9a-zA-Z_\-]{20}\b'''
+tags = ["gitlab"]
+keywords = [
+ "GR1348941",
+]
+
+[[rules]]
+id = "gitlab_runner_auth_token"
+description = "GitLab Runner Authentication Token"
+regex = '''\bglrt-[0-9a-zA-Z_\-]{20}\b'''
+tags = ["gitlab"]
+keywords = [
+ "glrt",
+]
+
+[[rules]]
+id = "gitlab_oauth_app_secret"
+description = "GitLab OAuth Application Secrets"
+regex = '''\bgloas-[0-9a-zA-Z_\-]{64}\b'''
+tags = ["gitlab"]
+keywords = [
+ "gloas",
+]
+
+[[rules]]
+id = "gitlab_feed_token_v2"
+description = "GitLab Feed Token"
+regex = '''\bglft-[0-9a-zA-Z_\-]{20}\b'''
+tags = ["gitlab"]
+keywords = [
+ "glft",
+]
+
+[[rules]]
+id = "gitlab_kubernetes_agent_token"
+description = "GitLab Agent for Kubernetes token"
+regex = '''\bglagent-[0-9a-zA-Z_\-]{50}\b'''
+tags = ["gitlab"]
+keywords = [
+ "glagent",
+]
+
+[[rules]]
+id = "gitlab_incoming_email_token"
+description = "GitLab Incoming email token"
+regex = '''\bglimt-[0-9a-zA-Z_\-]{25}\b'''
+tags = ["gitlab"]
+keywords = [
+ "glimt",
+]
+
+[[rules]]
+id = "AWS"
+description = "AWS Access Token"
+regex = '''\bAKIA[0-9A-Z]{16}\b'''
+tags = ["aws", "revocation_type"]
+keywords = [
+ "AKIA",
+]
+
+[[rules]]
+id = "Github Personal Access Token"
+description = "Github Personal Access Token"
+regex = '''ghp_[0-9a-zA-Z]{36}'''
+keywords = [
+ "ghp_",
+]
+
+[[rules]]
+id = "Github OAuth Access Token"
+description = "Github OAuth Access Token"
+regex = '''gho_[0-9a-zA-Z]{36}'''
+keywords = [
+ "gho_",
+]
+
+[[rules]]
+id = "Github App Token"
+description = "Github App Token"
+regex = '''(ghu|ghs)_[0-9a-zA-Z]{36}'''
+keywords = [
+ "ghu_",
+ "ghs_"
+]
+
+[[rules]]
+id = "Github Refresh Token"
+description = "Github Refresh Token"
+regex = '''ghr_[0-9a-zA-Z]{76}'''
+keywords = [
+ "ghr_"
+]
+
+[[rules]]
+id = "Shopify shared secret"
+description = "Shopify shared secret"
+regex = '''shpss_[a-fA-F0-9]{32}'''
+keywords = [
+ "shpss_"
+]
+
+[[rules]]
+id = "Shopify access token"
+description = "Shopify access token"
+regex = '''shpat_[a-fA-F0-9]{32}'''
+keywords = [
+ "shpat_"
+]
+
+[[rules]]
+id = "Shopify custom app access token"
+description = "Shopify custom app access token"
+regex = '''shpca_[a-fA-F0-9]{32}'''
+keywords = [
+ "shpca_"
+]
+
+[[rules]]
+id = "Shopify private app access token"
+description = "Shopify private app access token"
+regex = '''shppa_[a-fA-F0-9]{32}'''
+keywords = [
+ "shppa_"
+]
+
+[[rules]]
+id = "Slack token"
+description = "Slack token"
+regex = '''xox[baprs]-([0-9a-zA-Z]{10,48})'''
+keywords = [
+ "xoxb","xoxa","xoxp","xoxr","xoxs",
+]
+
+[[rules]]
+id = "Stripe"
+description = "Stripe"
+regex = '''(?i)(sk|pk)_(test|live)_[0-9a-z]{10,32}'''
+keywords = [
+ "sk_test","pk_test","sk_live","pk_live",
+]
+
+[[rules]]
+id = "PyPI upload token"
+description = "PyPI upload token"
+regex = '''pypi-AgEIcHlwaS5vcmc[A-Za-z0-9-_]{50,1000}'''
+tags = ["pypi", "revocation_type"]
+keywords = [
+ "pypi-AgEIcHlwaS5vcmc",
+]
+
+[[rules]]
+id = "Google (GCP) Service-account"
+description = "Google (GCP) Service-account"
+tags = ["gitlab_partner_token", "revocation_type"]
+regex = '''\"private_key\":\s*\"-{5}BEGIN PRIVATE KEY-{5}[\s\S]*?",'''
+keywords = [
+ "service_account",
+]
+
+[[rules]]
+id = "GCP API key"
+description = "GCP API keys can be misused to gain API quota from billed projects"
+tags = ["gitlab_partner_token", "revocation_type"]
+regex = '''(?i)\b(AIza[0-9A-Za-z-_]{35})(?:['|\"|\n|\r|\s|\x60|;]|$)'''
+secretGroup = 1
+keywords = [
+ "AIza",
+]
+
+[[rules]]
+id = "GCP OAuth client secret"
+description = "GCP OAuth client secrets can be misused to spoof your application"
+tags = ["gitlab_partner_token", "revocation_type"]
+regex = '''GOCSPX-[a-zA-Z0-9_-]{28}'''
+keywords = [
+ "GOCSPX-",
+]
+
+[[rules]]
+id = "Grafana API token"
+description = "Grafana API token"
+regex = '''['\"]eyJrIjoi(?i)[a-z0-9-_=]{72,92}['\"]'''
+keywords = [
+ "grafana",
+]
+
+[[rules]]
+id = "Hashicorp Terraform user/org API token"
+description = "Hashicorp Terraform user/org API token"
+regex = '''['\"](?i)[a-z0-9]{14}\.atlasv1\.[a-z0-9-_=]{60,70}['\"]'''
+keywords = [
+ "atlasv1",
+ "hashicorp",
+ "terraform"
+]
+
+[[rules]]
+id = "Hashicorp Vault batch token"
+description = "Hashicorp Vault batch token"
+regex = '''b\.AAAAAQ[0-9a-zA-Z_-]{156}'''
+keywords = [
+ "hashicorp",
+ "AAAAAQ",
+ "vault"
+]
+
+[[rules]]
+id = "Mailchimp API key"
+description = "Mailchimp API key"
+regex = '''(?i)(mailchimp[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-f0-9]{32}-us20)['\"]'''
+secretGroup = 3
+keywords = [
+ "mailchimp",
+]
+
+[[rules]]
+id = "Mailgun private API token"
+description = "Mailgun private API token"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"](key-[a-f0-9]{32})['\"]'''
+secretGroup = 3
+keywords = [
+ "mailgun",
+]
+
+[[rules]]
+id = "Mailgun webhook signing key"
+description = "Mailgun webhook signing key"
+regex = '''(?i)(mailgun[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([a-h0-9]{32}-[a-h0-9]{8}-[a-h0-9]{8})['\"]'''
+secretGroup = 3
+keywords = [
+ "mailgun",
+]
+
+[[rules]]
+id = "New Relic user API Key"
+description = "New Relic user API Key"
+regex = '''['\"](NRAK-[A-Z0-9]{27})['\"]'''
+keywords = [
+ "NRAK",
+]
+
+[[rules]]
+id = "New Relic user API ID"
+description = "New Relic user API ID"
+regex = '''(?i)(newrelic[a-z0-9_ .\-,]{0,25})(=|>|:=|\|\|:|<=|=>|:).{0,5}['\"]([A-Z0-9]{64})['\"]'''
+secretGroup = 3
+keywords = [
+ "newrelic",
+]
+
+[[rules]]
+id = "npm access token"
+description = "npm access token"
+regex = '''['\"](npm_(?i)[a-z0-9]{36})['\"]'''
+keywords = [
+ "npm_",
+]
+
+[[rules]]
+id = "Rubygem API token"
+description = "Rubygem API token"
+regex = '''rubygems_[a-f0-9]{48}'''
+keywords = [
+ "rubygems_",
+]
+
+[[rules]]
+id = "Segment Public API token"
+description = "Segment Public API token"
+regex = '''sgp_[a-zA-Z0-9]{64}'''
+keywords = [
+ "sgp_",
+]
+
+[[rules]]
+id = "Sendgrid API token"
+description = "Sendgrid API token"
+regex = '''SG\.(?i)[a-z0-9_\-\.]{66}'''
+keywords = [
+ "sendgrid",
+]
diff --git a/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb b/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb
new file mode 100644
index 00000000000..1e6ccf1e6a0
--- /dev/null
+++ b/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb
@@ -0,0 +1,222 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detection do
+ subject(:scan) { described_class.new }
+
+ def new_blob(id:, data:)
+ Struct.new(:id, :data).new(id, data)
+ end
+
+ let(:ruleset) do
+ {
+ "title" => "gitleaks config",
+ "rules" => [
+ {
+ "id" => "gitlab_personal_access_token",
+ "description" => "GitLab Personal Access Token",
+ "regex" => "glpat-[0-9a-zA-Z_\\-]{20}",
+ "tags" => %w[gitlab revocation_type],
+ "keywords" => ["glpat"]
+ },
+ {
+ "id" => "gitlab_pipeline_trigger_token",
+ "description" => "GitLab Pipeline Trigger Token",
+ "regex" => "glptt-[0-9a-zA-Z_\\-]{40}",
+ "tags" => ["gitlab"],
+ "keywords" => ["glptt"]
+ },
+ {
+ "id" => "gitlab_runner_registration_token",
+ "description" => "GitLab Runner Registration Token",
+ "regex" => "GR1348941[0-9a-zA-Z_-]{20}",
+ "tags" => ["gitlab"],
+ "keywords" => ["GR1348941"]
+ },
+ {
+ "id" => "gitlab_feed_token_v2",
+ "description" => "GitLab Feed Token",
+ "regex" => "glft-[0-9a-zA-Z_-]{20}",
+ "tags" => ["gitlab"],
+ "keywords" => ["glft"]
+ }
+ ]
+ }
+ end
+
+ it "does not raise an error parsing the toml file" do
+ expect { scan }.not_to raise_error
+ end
+
+ context "when it creates RE2 patterns from file data" do
+ before do
+ allow(scan).to receive(:parse_ruleset).and_return(ruleset)
+ end
+
+ it "does not raise an error when building patterns" do
+ expect { scan }.not_to raise_error
+ end
+ end
+
+ context "when matching patterns" do
+ before do
+ allow(scan).to receive(:parse_ruleset).and_return(ruleset)
+ end
+
+ context 'when the blob does not contain a secret' do
+ let(:blobs) do
+ [
+ new_blob(id: 1234, data: "no secrets")
+ ]
+ end
+
+ it "does not match" do
+ expected_response = Gitlab::SecretDetection::Response.new(Gitlab::SecretDetection::Status::NOT_FOUND)
+
+ expect(scan.secrets_scan(blobs)).to eq(expected_response)
+ end
+
+ it "attempts to keyword match returning no blobs for further scan" do
+ expect(scan).to receive(:filter_by_keywords)
+ .with(blobs)
+ .and_return([])
+
+ scan.secrets_scan(blobs)
+ end
+
+ it "does not attempt to regex match" do
+ expect(scan).not_to receive(:match_rules_bulk)
+
+ scan.secrets_scan(blobs)
+ end
+ end
+
+ context "when multiple blobs contains secrets" do
+ let(:blobs) do
+ [
+ new_blob(id: 111, data: "glpat-12312312312312312312"), # gitleaks:allow
+ new_blob(id: 222, data: "\n\nglptt-1231231231231231231212312312312312312312"), # gitleaks:allow
+ new_blob(id: 333, data: "data with no secret"),
+ new_blob(id: 444, data: "GR134894112312312312312312312\nglft-12312312312312312312") # gitleaks:allow
+ ]
+ end
+
+ it "matches different types of rules" do
+ expected_response = Gitlab::SecretDetection::Response.new(
+ Gitlab::SecretDetection::Status::FOUND,
+ [
+ Gitlab::SecretDetection::Finding.new(
+ blobs[0].id,
+ Gitlab::SecretDetection::Status::FOUND,
+ 1,
+ ruleset['rules'][0]['id'],
+ ruleset['rules'][0]['description']
+ ),
+ Gitlab::SecretDetection::Finding.new(
+ blobs[1].id,
+ Gitlab::SecretDetection::Status::FOUND,
+ 3,
+ ruleset['rules'][1]['id'],
+ ruleset['rules'][1]['description']
+ ),
+ Gitlab::SecretDetection::Finding.new(
+ blobs[3].id,
+ Gitlab::SecretDetection::Status::FOUND,
+ 1,
+ ruleset['rules'][2]['id'],
+ ruleset['rules'][2]['description']
+ ),
+ Gitlab::SecretDetection::Finding.new(
+ blobs[3].id,
+ Gitlab::SecretDetection::Status::FOUND,
+ 2,
+ ruleset['rules'][3]['id'],
+ ruleset['rules'][3]['description']
+ )
+ ]
+ )
+
+ expect(scan.secrets_scan(blobs)).to eq(expected_response)
+ end
+ end
+
+ context "when configured with time out" do
+ let(:each_blob_timeout_secs) { 0.000_001 } # 1 micro-sec to intentionally timeout large blob
+
+ let(:large_data) do
+ ("large data with a secret glpat-12312312312312312312\n" * 10_000_000).freeze # gitleaks:allow
+ end
+
+ let(:blobs) do
+ [
+ new_blob(id: 111, data: "GR134894112312312312312312312"), # gitleaks:allow
+ new_blob(id: 333, data: "data with no secret"),
+ new_blob(id: 333, data: large_data)
+ ]
+ end
+
+ let(:all_large_blobs) do
+ [
+ new_blob(id: 111, data: large_data),
+ new_blob(id: 222, data: large_data),
+ new_blob(id: 333, data: large_data)
+ ]
+ end
+
+ it "whole secret detection scan operation times out" do
+ scan_timeout_secs = 0.000_001 # 1 micro-sec to intentionally timeout large blob
+
+ response = Gitlab::SecretDetection::Response.new(Gitlab::SecretDetection::Status::SCAN_TIMEOUT)
+
+ expect(scan.secrets_scan(blobs, timeout: scan_timeout_secs)).to eq(response)
+ end
+
+ it "one of the blobs times out while others continue to get scanned" do
+ expected_response = Gitlab::SecretDetection::Response.new(
+ Gitlab::SecretDetection::Status::FOUND_WITH_ERRORS,
+ [
+ Gitlab::SecretDetection::Finding.new(
+ blobs[0].id,
+ Gitlab::SecretDetection::Status::FOUND,
+ 1,
+ ruleset['rules'][2]['id'],
+ ruleset['rules'][2]['description']
+ ),
+ Gitlab::SecretDetection::Finding.new(
+ blobs[2].id,
+ Gitlab::SecretDetection::Status::BLOB_TIMEOUT
+ )
+ ]
+ )
+
+ expect(scan.secrets_scan(blobs, blob_timeout: each_blob_timeout_secs)).to eq(expected_response)
+ end
+
+ it "all the blobs time out" do
+ # scan status changes to SCAN_TIMEOUT when *all* the blobs time out
+ expected_scan_status = Gitlab::SecretDetection::Status::SCAN_TIMEOUT
+
+ expected_response = Gitlab::SecretDetection::Response.new(
+ expected_scan_status,
+ [
+ Gitlab::SecretDetection::Finding.new(
+ all_large_blobs[0].id,
+ Gitlab::SecretDetection::Status::BLOB_TIMEOUT
+ ),
+ Gitlab::SecretDetection::Finding.new(
+ all_large_blobs[1].id,
+ Gitlab::SecretDetection::Status::BLOB_TIMEOUT
+ ),
+ Gitlab::SecretDetection::Finding.new(
+ all_large_blobs[2].id,
+ Gitlab::SecretDetection::Status::BLOB_TIMEOUT
+ )
+ ]
+ )
+
+ expect(scan.secrets_scan(all_large_blobs, blob_timeout: each_blob_timeout_secs)).to eq(expected_response)
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-secret_detection/spec/spec_helper.rb b/gems/gitlab-secret_detection/spec/spec_helper.rb
new file mode 100644
index 00000000000..b694e52d2b6
--- /dev/null
+++ b/gems/gitlab-secret_detection/spec/spec_helper.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'gitlab/secret_detection'
+require 'rspec-parameterized'
+
+RSpec.configure do |config|
+ # Enable flags like --only-failures and --next-failure
+ config.example_status_persistence_file_path = ".rspec_status"
+
+ # Disable RSpec exposing methods globally on `Module` and `main`
+ config.disable_monkey_patching!
+
+ Dir['./spec/support/**/*.rb'].each { |f| require f }
+
+ config.expect_with :rspec do |c|
+ c.syntax = :expect
+ end
+end
diff --git a/gems/gitlab-utils/Gemfile.lock b/gems/gitlab-utils/Gemfile.lock
index e6cfe03e60e..ef7c2d57c7a 100644
--- a/gems/gitlab-utils/Gemfile.lock
+++ b/gems/gitlab-utils/Gemfile.lock
@@ -2,6 +2,7 @@ PATH
remote: ../gitlab-rspec
specs:
gitlab-rspec (0.1.0)
+ activerecord (>= 6.1, < 8)
activesupport (>= 6.1, < 8)
rspec (~> 3.0)
@@ -31,6 +32,11 @@ GEM
erubi (~> 1.4)
rails-dom-testing (~> 2.0)
rails-html-sanitizer (~> 1.1, >= 1.2.0)
+ activemodel (7.0.5)
+ activesupport (= 7.0.5)
+ activerecord (7.0.5)
+ activemodel (= 7.0.5)
+ activesupport (= 7.0.5)
activesupport (7.0.5)
concurrent-ruby (~> 1.0, >= 1.0.2)
i18n (>= 1.6, < 2)
diff --git a/gems/gitlab-utils/lib/gitlab/utils.rb b/gems/gitlab-utils/lib/gitlab/utils.rb
index 4e08ee8fcaf..b7cb63210ee 100644
--- a/gems/gitlab-utils/lib/gitlab/utils.rb
+++ b/gems/gitlab-utils/lib/gitlab/utils.rb
@@ -8,6 +8,7 @@ module Gitlab
module Utils
extend self
DoubleEncodingError = Class.new(StandardError)
+ ConcurrentRubyThreadIsUsedError = Class.new(StandardError)
def allowlisted?(absolute_path, allowlist)
path = absolute_path.downcase
@@ -259,5 +260,23 @@ module Gitlab
untrimmed = trimmed
end
end
+
+ def restrict_within_concurrent_ruby
+ previous = Thread.current[:restrict_within_concurrent_ruby]
+ Thread.current[:restrict_within_concurrent_ruby] = true
+
+ yield
+ ensure
+ Thread.current[:restrict_within_concurrent_ruby] = previous
+ end
+
+ # Running external methods can allocate I/O bound resources (like PostgreSQL connection or Gitaly)
+ # This is forbidden when running within a concurrent Ruby thread, for example `async` HTTP requests
+ # provided by the `gitlab-http` gem.
+ def raise_if_concurrent_ruby!(what)
+ return unless Thread.current[:restrict_within_concurrent_ruby]
+
+ raise ConcurrentRubyThreadIsUsedError, "Cannot run '#{what}' if running from `Concurrent::Promise`."
+ end
end
end
diff --git a/gems/gitlab-utils/lib/gitlab/utils/all.rb b/gems/gitlab-utils/lib/gitlab/utils/all.rb
index 200a21aad88..2e306e7c763 100644
--- a/gems/gitlab-utils/lib/gitlab/utils/all.rb
+++ b/gems/gitlab-utils/lib/gitlab/utils/all.rb
@@ -4,3 +4,4 @@ require_relative "../utils"
require_relative "../version_info"
require_relative "version"
require_relative "strong_memoize"
+require_relative "system"
diff --git a/gems/gitlab-utils/lib/gitlab/utils/system.rb b/gems/gitlab-utils/lib/gitlab/utils/system.rb
new file mode 100644
index 00000000000..ada9da005b3
--- /dev/null
+++ b/gems/gitlab-utils/lib/gitlab/utils/system.rb
@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Utils
+ # Module for gathering system/process statistics such as the memory usage.
+ #
+ # This module relies on the /proc filesystem being available. If /proc is
+ # not available the methods of this module will be stubbed.
+ module System
+ extend self
+
+ PROC_STAT_PATH = '/proc/self/stat'
+ PROC_STATUS_PATH = '/proc/%s/status'
+ PROC_SMAPS_ROLLUP_PATH = '/proc/%s/smaps_rollup'
+ PROC_LIMITS_PATH = '/proc/self/limits'
+ PROC_FD_GLOB = '/proc/self/fd/*'
+ PROC_MEM_INFO = '/proc/meminfo'
+
+ PRIVATE_PAGES_PATTERN = /^(?<type>Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/
+ PSS_PATTERN = /^Pss:\s+(?<value>\d+)/
+ RSS_TOTAL_PATTERN = /^VmRSS:\s+(?<value>\d+)/
+ RSS_ANON_PATTERN = /^RssAnon:\s+(?<value>\d+)/
+ RSS_FILE_PATTERN = /^RssFile:\s+(?<value>\d+)/
+ MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/
+ MEM_TOTAL_PATTERN = /^MemTotal:\s+(?<value>\d+) (?<unit>.+)/
+
+ def summary
+ proportional_mem = memory_usage_uss_pss
+ {
+ version: RUBY_DESCRIPTION,
+ gc_stat: GC.stat,
+ memory_rss: memory_usage_rss[:total],
+ memory_uss: proportional_mem[:uss],
+ memory_pss: proportional_mem[:pss],
+ time_cputime: cpu_time,
+ time_realtime: real_time,
+ time_monotonic: monotonic_time
+ }
+ end
+
+ # Returns the given process' RSS (resident set size) in bytes.
+ def memory_usage_rss(pid: 'self')
+ results = { total: 0, anon: 0, file: 0 }
+
+ safe_yield_procfile(PROC_STATUS_PATH % pid) do |io|
+ io.each_line do |line|
+ if (value = parse_metric_value(line, RSS_TOTAL_PATTERN)) > 0
+ results[:total] = value.kilobytes
+ elsif (value = parse_metric_value(line, RSS_ANON_PATTERN)) > 0
+ results[:anon] = value.kilobytes
+ elsif (value = parse_metric_value(line, RSS_FILE_PATTERN)) > 0
+ results[:file] = value.kilobytes
+ end
+ end
+ end
+
+ results
+ end
+
+ # Returns the given process' USS/PSS (unique/proportional set size) in bytes.
+ def memory_usage_uss_pss(pid: 'self')
+ sum_matches(PROC_SMAPS_ROLLUP_PATH % pid, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN)
+ .transform_values(&:kilobytes)
+ end
+
+ def memory_total
+ sum_matches(PROC_MEM_INFO, memory_total: MEM_TOTAL_PATTERN)[:memory_total].kilobytes
+ end
+
+ def file_descriptor_count
+ Dir.glob(PROC_FD_GLOB).length
+ end
+
+ def max_open_file_descriptors
+ sum_matches(PROC_LIMITS_PATH, max_fds: MAX_OPEN_FILES_PATTERN)[:max_fds]
+ end
+
+ def cpu_time
+ Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID, :float_second)
+ end
+
+ # Returns the current real time in a given precision.
+ #
+ # Returns the time as a Float for precision = :float_second.
+ def real_time(precision = :float_second)
+ Process.clock_gettime(Process::CLOCK_REALTIME, precision)
+ end
+
+ # Returns the current monotonic clock time as seconds with microseconds precision.
+ #
+ # Returns the time as a Float.
+ def monotonic_time
+ Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_second)
+ end
+
+ def thread_cpu_time
+ # Not all OS kernels are supporting `Process::CLOCK_THREAD_CPUTIME_ID`
+ # Refer: https://gitlab.com/gitlab-org/gitlab/issues/30567#note_221765627
+ return unless defined?(Process::CLOCK_THREAD_CPUTIME_ID)
+
+ Process.clock_gettime(Process::CLOCK_THREAD_CPUTIME_ID, :float_second)
+ end
+
+ def thread_cpu_duration(start_time)
+ end_time = thread_cpu_time
+ return unless start_time && end_time
+
+ end_time - start_time
+ end
+
+ # Returns the total time the current process has been running in seconds.
+ def process_runtime_elapsed_seconds
+ # Entry 22 (1-indexed) contains the process `starttime`, see:
+ # https://man7.org/linux/man-pages/man5/proc.5.html
+ #
+ # This value is a fixed timestamp in clock ticks.
+ # To obtain an elapsed time in seconds, we divide by the number
+ # of ticks per second and subtract from the system uptime.
+ start_time_ticks = proc_stat_entries[21].to_f
+ clock_ticks_per_second = Etc.sysconf(Etc::SC_CLK_TCK)
+ uptime - (start_time_ticks / clock_ticks_per_second)
+ end
+
+ private
+
+ # Given a path to a file in /proc and a hash of (metric, pattern) pairs,
+ # sums up all values found for those patterns under the respective metric.
+ def sum_matches(proc_file, **patterns)
+ results = patterns.transform_values { 0 }
+
+ safe_yield_procfile(proc_file) do |io|
+ io.each_line do |line|
+ patterns.each do |metric, pattern|
+ results[metric] += parse_metric_value(line, pattern)
+ end
+ end
+ end
+
+ results
+ end
+
+ def parse_metric_value(line, pattern)
+ match = line.match(pattern)
+ return 0 unless match
+
+ match.named_captures.fetch('value', 0).to_i
+ end
+
+ def proc_stat_entries
+ safe_yield_procfile(PROC_STAT_PATH) do |io|
+ io.read.split(' ')
+ end || []
+ end
+
+ def safe_yield_procfile(path, &block)
+ File.open(path, &block)
+ rescue Errno::ENOENT
+ # This means the procfile we're reading from did not exist;
+ # most likely we're on Darwin.
+ end
+
+ # Equivalent to reading /proc/uptime on Linux 2.6+.
+ #
+ # Returns 0 if not supported, e.g. on Darwin.
+ def uptime
+ Process.clock_gettime(Process::CLOCK_BOOTTIME)
+ rescue NameError
+ 0
+ end
+ end
+ end
+end
diff --git a/gems/gitlab-utils/spec/gitlab/utils/system_spec.rb b/gems/gitlab-utils/spec/gitlab/utils/system_spec.rb
new file mode 100644
index 00000000000..6d4f3bf039c
--- /dev/null
+++ b/gems/gitlab-utils/spec/gitlab/utils/system_spec.rb
@@ -0,0 +1,364 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Utils::System do
+ context 'when /proc files exist' do
+ # Modified column 22 to be 1000 (starttime ticks)
+ let(:proc_stat) do
+ <<~SNIP
+ 2095 (ruby) R 0 2095 2095 34818 2095 4194560 211267 7897 2 0 287 51 10 1 20 0 5 0 1000 566210560 80885 18446744073709551615 94736211292160 94736211292813 140720919612064 0 0 0 0 0 1107394127 0 0 0 17 3 0 0 0 0 0 94736211303768 94736211304544 94736226689024 140720919619473 140720919619513 140720919619513 140720919621604 0
+ SNIP
+ end
+
+ # Fixtures pulled from:
+ # Linux carbon 5.3.0-7648-generic #41~1586789791~19.10~9593806-Ubuntu SMP
+ # Mon Apr 13 17:50:40 UTC x86_64 x86_64 x86_64 GNU/Linux
+ let(:proc_status) do
+ # most rows omitted for brevity
+ <<~SNIP
+ Name: less
+ VmHWM: 2468 kB
+ VmRSS: 2468 kB
+ RssAnon: 260 kB
+ RssFile: 1024 kB
+ SNIP
+ end
+
+ let(:proc_smaps_rollup) do
+ # full snapshot
+ <<~SNIP
+ Rss: 2564 kB
+ Pss: 503 kB
+ Pss_Anon: 312 kB
+ Pss_File: 191 kB
+ Pss_Shmem: 0 kB
+ Shared_Clean: 2100 kB
+ Shared_Dirty: 0 kB
+ Private_Clean: 152 kB
+ Private_Dirty: 312 kB
+ Referenced: 2564 kB
+ Anonymous: 312 kB
+ LazyFree: 0 kB
+ AnonHugePages: 0 kB
+ ShmemPmdMapped: 0 kB
+ Shared_Hugetlb: 0 kB
+ Private_Hugetlb: 0 kB
+ Swap: 0 kB
+ SwapPss: 0 kB
+ Locked: 0 kB
+ SNIP
+ end
+
+ let(:proc_limits) do
+ # full snapshot
+ <<~SNIP
+ Limit Soft Limit Hard Limit Units
+ Max cpu time unlimited unlimited seconds
+ Max file size unlimited unlimited bytes
+ Max data size unlimited unlimited bytes
+ Max stack size 8388608 unlimited bytes
+ Max core file size 0 unlimited bytes
+ Max resident set unlimited unlimited bytes
+ Max processes 126519 126519 processes
+ Max open files 1024 1048576 files
+ Max locked memory 67108864 67108864 bytes
+ Max address space unlimited unlimited bytes
+ Max file locks unlimited unlimited locks
+ Max pending signals 126519 126519 signals
+ Max msgqueue size 819200 819200 bytes
+ Max nice priority 0 0
+ Max realtime priority 0 0
+ Max realtime timeout unlimited unlimited us
+ SNIP
+ end
+
+ let(:mem_info) do
+ # full snapshot
+ <<~SNIP
+ MemTotal: 15362536 kB
+ MemFree: 3403136 kB
+ MemAvailable: 13044528 kB
+ Buffers: 272188 kB
+ Cached: 8171312 kB
+ SwapCached: 0 kB
+ Active: 3332084 kB
+ Inactive: 6981076 kB
+ Active(anon): 1603868 kB
+ Inactive(anon): 9044 kB
+ Active(file): 1728216 kB
+ Inactive(file): 6972032 kB
+ Unevictable: 18676 kB
+ Mlocked: 18676 kB
+ SwapTotal: 0 kB
+ SwapFree: 0 kB
+ Dirty: 6808 kB
+ Writeback: 0 kB
+ AnonPages: 1888300 kB
+ Mapped: 166164 kB
+ Shmem: 12932 kB
+ KReclaimable: 1275120 kB
+ Slab: 1495480 kB
+ SReclaimable: 1275120 kB
+ SUnreclaim: 220360 kB
+ KernelStack: 7072 kB
+ PageTables: 11936 kB
+ NFS_Unstable: 0 kB
+ Bounce: 0 kB
+ WritebackTmp: 0 kB
+ CommitLimit: 7681268 kB
+ Committed_AS: 4976100 kB
+ VmallocTotal: 34359738367 kB
+ VmallocUsed: 25532 kB
+ VmallocChunk: 0 kB
+ Percpu: 23200 kB
+ HardwareCorrupted: 0 kB
+ AnonHugePages: 202752 kB
+ ShmemHugePages: 0 kB
+ ShmemPmdMapped: 0 kB
+ FileHugePages: 0 kB
+ FilePmdMapped: 0 kB
+ CmaTotal: 0 kB
+ CmaFree: 0 kB
+ HugePages_Total: 0
+ HugePages_Free: 0
+ HugePages_Rsvd: 0
+ HugePages_Surp: 0
+ Hugepagesize: 2048 kB
+ Hugetlb: 0 kB
+ DirectMap4k: 4637504 kB
+ DirectMap2M: 11087872 kB
+ DirectMap1G: 2097152 kB
+ SNIP
+ end
+
+ describe '.memory_usage_rss' do
+ context 'without PID' do
+ it "returns a hash containing RSS metrics in bytes for current process" do
+ mock_existing_proc_file('/proc/self/status', proc_status)
+
+ expect(described_class.memory_usage_rss).to eq(
+ total: 2527232,
+ anon: 266240,
+ file: 1048576
+ )
+ end
+ end
+
+ context 'with PID' do
+ it "returns a hash containing RSS metrics in bytes for given process" do
+ mock_existing_proc_file('/proc/7/status', proc_status)
+
+ expect(described_class.memory_usage_rss(pid: 7)).to eq(
+ total: 2527232,
+ anon: 266240,
+ file: 1048576
+ )
+ end
+ end
+ end
+
+ describe '.file_descriptor_count' do
+ it 'returns the amount of open file descriptors' do
+ expect(Dir).to receive(:glob).and_return(['/some/path', '/some/other/path'])
+
+ expect(described_class.file_descriptor_count).to eq(2)
+ end
+ end
+
+ describe '.max_open_file_descriptors' do
+ it 'returns the max allowed open file descriptors' do
+ mock_existing_proc_file('/proc/self/limits', proc_limits)
+
+ expect(described_class.max_open_file_descriptors).to eq(1024)
+ end
+ end
+
+ describe '.memory_usage_uss_pss' do
+ context 'without PID' do
+ it "returns the current process' unique and porportional set size (USS/PSS) in bytes" do
+ mock_existing_proc_file('/proc/self/smaps_rollup', proc_smaps_rollup)
+
+ # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024
+ expect(described_class.memory_usage_uss_pss).to eq(uss: 475136, pss: 515072)
+ end
+ end
+
+ context 'with PID' do
+ it "returns the given process' unique and porportional set size (USS/PSS) in bytes" do
+ mock_existing_proc_file('/proc/7/smaps_rollup', proc_smaps_rollup)
+
+ # (Private_Clean (152 kB) + Private_Dirty (312 kB) + Private_Hugetlb (0 kB)) * 1024
+ expect(described_class.memory_usage_uss_pss(pid: 7)).to eq(uss: 475136, pss: 515072)
+ end
+ end
+ end
+
+ describe '.memory_total' do
+ it "returns the current process' resident set size (RSS) in bytes" do
+ mock_existing_proc_file('/proc/meminfo', mem_info)
+
+ expect(described_class.memory_total).to eq(15731236864)
+ end
+ end
+
+ describe '.process_runtime_elapsed_seconds' do
+ it 'returns the seconds elapsed since the process was started' do
+ # sets process starttime ticks to 1000
+ mock_existing_proc_file('/proc/self/stat', proc_stat)
+ # system clock ticks/sec
+ expect(Etc).to receive(:sysconf).with(Etc::SC_CLK_TCK).and_return(100)
+ # system uptime in seconds
+ expect(::Process).to receive(:clock_gettime).and_return(15)
+
+ # uptime - (starttime_ticks / ticks_per_sec)
+ expect(described_class.process_runtime_elapsed_seconds).to eq(5)
+ end
+
+ context 'when inputs are not available' do
+ it 'returns 0' do
+ mock_missing_proc_file
+ expect(::Process).to receive(:clock_gettime).and_raise(NameError)
+
+ expect(described_class.process_runtime_elapsed_seconds).to eq(0)
+ end
+ end
+ end
+
+ describe '.summary' do
+ it 'contains a selection of the available fields' do
+ stub_const('RUBY_DESCRIPTION', 'ruby-3.0-patch1')
+ mock_existing_proc_file('/proc/self/status', proc_status)
+ mock_existing_proc_file('/proc/self/smaps_rollup', proc_smaps_rollup)
+
+ summary = described_class.summary
+
+ expect(summary[:version]).to eq('ruby-3.0-patch1')
+ expect(summary[:gc_stat].keys).to eq(GC.stat.keys)
+ expect(summary[:memory_rss]).to eq(2527232)
+ expect(summary[:memory_uss]).to eq(475136)
+ expect(summary[:memory_pss]).to eq(515072)
+ expect(summary[:time_cputime]).to be_a(Float)
+ expect(summary[:time_realtime]).to be_a(Float)
+ expect(summary[:time_monotonic]).to be_a(Float)
+ end
+ end
+ end
+
+ context 'when /proc files do not exist' do
+ before do
+ mock_missing_proc_file
+ end
+
+ describe '.memory_usage_rss' do
+ it 'returns 0 for all components' do
+ expect(described_class.memory_usage_rss).to eq(
+ total: 0,
+ anon: 0,
+ file: 0
+ )
+ end
+ end
+
+ describe '.memory_usage_uss_pss' do
+ it "returns 0 for all components" do
+ expect(described_class.memory_usage_uss_pss).to eq(uss: 0, pss: 0)
+ end
+ end
+
+ describe '.file_descriptor_count' do
+ it 'returns 0' do
+ expect(Dir).to receive(:glob).and_return([])
+
+ expect(described_class.file_descriptor_count).to eq(0)
+ end
+ end
+
+ describe '.max_open_file_descriptors' do
+ it 'returns 0' do
+ expect(described_class.max_open_file_descriptors).to eq(0)
+ end
+ end
+
+ describe '.summary' do
+ it 'returns only available fields' do
+ summary = described_class.summary
+
+ expect(summary[:version]).to be_a(String)
+ expect(summary[:gc_stat].keys).to eq(GC.stat.keys)
+ expect(summary[:memory_rss]).to eq(0)
+ expect(summary[:memory_uss]).to eq(0)
+ expect(summary[:memory_pss]).to eq(0)
+ expect(summary[:time_cputime]).to be_a(Float)
+ expect(summary[:time_realtime]).to be_a(Float)
+ expect(summary[:time_monotonic]).to be_a(Float)
+ end
+ end
+ end
+
+ describe '.cpu_time' do
+ it 'returns a Float' do
+ expect(described_class.cpu_time).to be_an(Float)
+ end
+ end
+
+ describe '.real_time' do
+ it 'returns a Float' do
+ expect(described_class.real_time).to be_an(Float)
+ end
+ end
+
+ describe '.monotonic_time' do
+ it 'returns a Float' do
+ expect(described_class.monotonic_time).to be_an(Float)
+ end
+ end
+
+ describe '.thread_cpu_time' do
+ it 'returns cpu_time on supported platform' do
+ stub_const("Process::CLOCK_THREAD_CPUTIME_ID", 16)
+
+ expect(Process).to receive(:clock_gettime)
+ .with(16, kind_of(Symbol)).and_return(0.111222333)
+
+ expect(described_class.thread_cpu_time).to eq(0.111222333)
+ end
+
+ it 'returns nil on unsupported platform' do
+ hide_const("Process::CLOCK_THREAD_CPUTIME_ID")
+
+ expect(described_class.thread_cpu_time).to be_nil
+ end
+ end
+
+ describe '.thread_cpu_duration' do
+ let(:start_time) { described_class.thread_cpu_time }
+
+ it 'returns difference between start and current time' do
+ stub_const("Process::CLOCK_THREAD_CPUTIME_ID", 16)
+
+ expect(Process).to receive(:clock_gettime)
+ .with(16, kind_of(Symbol))
+ .and_return(
+ 0.111222333,
+ 0.222333833
+ )
+
+ expect(described_class.thread_cpu_duration(start_time)).to eq(0.1111115)
+ end
+
+ it 'returns nil on unsupported platform' do
+ hide_const("Process::CLOCK_THREAD_CPUTIME_ID")
+
+ expect(described_class.thread_cpu_duration(start_time)).to be_nil
+ end
+ end
+
+ def mock_existing_proc_file(path, content)
+ allow(File).to receive(:open).with(path) { |_path, &block| block.call(StringIO.new(content)) }
+ end
+
+ def mock_missing_proc_file
+ allow(File).to receive(:open).and_raise(Errno::ENOENT)
+ end
+end
diff --git a/gems/gitlab-utils/spec/gitlab/utils_spec.rb b/gems/gitlab-utils/spec/gitlab/utils_spec.rb
index 53593190eea..69531225eef 100644
--- a/gems/gitlab-utils/spec/gitlab/utils_spec.rb
+++ b/gems/gitlab-utils/spec/gitlab/utils_spec.rb
@@ -476,4 +476,33 @@ RSpec.describe Gitlab::Utils, feature_category: :shared do
it { expect(described_class.valid_brackets?(input, allow_nested: allow_nested)).to eq(valid) }
end
end
+
+ describe '.restrict_within_concurrent_ruby' do
+ it 'assigns restrict_within_concurrent_ruby to the current thread and ensures it restores' do
+ expect(Thread.current[:restrict_within_concurrent_ruby]).to be_nil
+
+ described_class.restrict_within_concurrent_ruby do
+ expect(Thread.current[:restrict_within_concurrent_ruby]).to be(true)
+ end
+
+ expect(Thread.current[:restrict_within_concurrent_ruby]).to be_nil
+ end
+ end
+
+ describe '.raise_if_concurrent_ruby!' do
+ subject(:raise_if_concurrent_ruby!) { described_class.raise_if_concurrent_ruby!('test') }
+
+ it 'raises an exception when running within a concurrent Ruby thread' do
+ block = proc do
+ expect { raise_if_concurrent_ruby! }.to raise_error(Gitlab::Utils::ConcurrentRubyThreadIsUsedError,
+ "Cannot run 'test' if running from `Concurrent::Promise`.")
+ end
+
+ described_class.restrict_within_concurrent_ruby(&block)
+ end
+
+ it 'does not raise an exception when not running within a concurrent Ruby thread' do
+ expect { raise_if_concurrent_ruby! }.not_to raise_error
+ end
+ end
end
diff --git a/gems/rspec_flaky/lib/rspec_flaky/config.rb b/gems/rspec_flaky/lib/rspec_flaky/config.rb
deleted file mode 100644
index ca57d1e08be..00000000000
--- a/gems/rspec_flaky/lib/rspec_flaky/config.rb
+++ /dev/null
@@ -1,21 +0,0 @@
-# frozen_string_literal: true
-
-module RspecFlaky
- class Config
- def self.generate_report?
- !!(ENV['FLAKY_RSPEC_GENERATE_REPORT'] =~ /1|true/)
- end
-
- def self.suite_flaky_examples_report_path
- ENV['FLAKY_RSPEC_SUITE_REPORT_PATH'] || "rspec/flaky/suite-report.json"
- end
-
- def self.flaky_examples_report_path
- ENV['FLAKY_RSPEC_REPORT_PATH'] || "rspec/flaky/report.json"
- end
-
- def self.new_flaky_examples_report_path
- ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || "rspec/flaky/new-report.json"
- end
- end
-end
diff --git a/gems/rspec_flaky/lib/rspec_flaky/example.rb b/gems/rspec_flaky/lib/rspec_flaky/example.rb
deleted file mode 100644
index 4a128a151dc..00000000000
--- a/gems/rspec_flaky/lib/rspec_flaky/example.rb
+++ /dev/null
@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-require 'forwardable'
-require 'digest'
-
-module RspecFlaky
- # This is a wrapper class for RSpec::Core::Example
- class Example
- extend Forwardable
-
- def_delegators :execution_result, :status, :exception
-
- def initialize(rspec_example)
- @rspec_example = rspec_example.respond_to?(:example) ? rspec_example.example : rspec_example
- end
-
- def uid
- @uid ||= Digest::MD5.hexdigest("#{description}-#{file}") # rubocop:disable Fips/MD5
- end
-
- def example_id
- rspec_example.id
- end
-
- def file
- metadata[:file_path]
- end
-
- def line
- metadata[:line_number]
- end
-
- def description
- metadata[:full_description]
- end
-
- def attempts
- rspec_example.respond_to?(:attempts) ? rspec_example.attempts : 1
- end
-
- def feature_category
- metadata[:feature_category]
- end
-
- def to_h
- {
- example_id: example_id,
- file: file,
- line: line,
- description: description,
- last_attempts_count: attempts,
- feature_category: feature_category
- }
- end
-
- private
-
- attr_reader :rspec_example
-
- def metadata
- rspec_example.metadata
- end
-
- def execution_result
- rspec_example.execution_result
- end
- end
-end
diff --git a/gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb b/gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb
deleted file mode 100644
index 35d1f34d2a2..00000000000
--- a/gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-require 'ostruct'
-
-module RspecFlaky
- # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file
- class FlakyExample
- ALLOWED_ATTRIBUTES = %i[
- example_id
- file
- line
- description
- first_flaky_at
- last_flaky_at
- last_flaky_job
- last_attempts_count
- flaky_reports
- feature_category
- ].freeze
-
- def initialize(example_hash)
- @attributes = {
- first_flaky_at: Time.now,
- last_flaky_at: Time.now,
- last_flaky_job: nil,
- last_attempts_count: example_hash[:attempts],
- flaky_reports: 0,
- feature_category: example_hash[:feature_category]
- }.merge(example_hash.slice(*ALLOWED_ATTRIBUTES))
-
- %i[first_flaky_at last_flaky_at].each do |attr|
- attributes[attr] = Time.parse(attributes[attr]) if attributes[attr].is_a?(String)
- end
- end
-
- def update!(example_hash)
- attributes[:file] = example_hash[:file]
- attributes[:line] = example_hash[:line]
- attributes[:description] = example_hash[:description]
- attributes[:first_flaky_at] ||= Time.now
- attributes[:last_flaky_at] = Time.now
- attributes[:flaky_reports] += 1
- attributes[:feature_category] = example_hash[:feature_category]
- attributes[:last_attempts_count] = example_hash[:last_attempts_count] if example_hash[:last_attempts_count]
-
- return unless ENV['CI_JOB_URL']
-
- attributes[:last_flaky_job] = (ENV['CI_JOB_URL']).to_s
- end
-
- def to_h
- attributes.dup
- end
-
- private
-
- attr_reader :attributes
- end
-end
diff --git a/gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb b/gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb
deleted file mode 100644
index f03fe63d11b..00000000000
--- a/gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb
+++ /dev/null
@@ -1,34 +0,0 @@
-# frozen_string_literal: true
-
-require 'active_support/hash_with_indifferent_access'
-require 'delegate'
-
-require_relative 'flaky_example'
-
-module RspecFlaky
- class FlakyExamplesCollection < SimpleDelegator
- def initialize(collection = {})
- raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" unless collection.is_a?(Hash)
-
- collection_of_flaky_examples =
- collection.map do |uid, example|
- [
- uid,
- FlakyExample.new(example.to_h.symbolize_keys)
- ]
- end
-
- super(Hash[collection_of_flaky_examples])
- end
-
- def to_h
- transform_values(&:to_h).deep_symbolize_keys
- end
-
- def -(other)
- raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!" unless other.respond_to?(:key)
-
- self.class.new(reject { |uid, _| other.key?(uid) })
- end
- end
-end
diff --git a/gems/rspec_flaky/lib/rspec_flaky/listener.rb b/gems/rspec_flaky/lib/rspec_flaky/listener.rb
deleted file mode 100644
index c2deb1c327a..00000000000
--- a/gems/rspec_flaky/lib/rspec_flaky/listener.rb
+++ /dev/null
@@ -1,70 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-
-require_relative 'config'
-require_relative 'example'
-require_relative 'flaky_example'
-require_relative 'flaky_examples_collection'
-require_relative 'report'
-
-module RspecFlaky
- class Listener
- # - suite_flaky_examples: contains all the currently tracked flacky example
- # for the whole RSpec suite
- # - flaky_examples: contains the examples detected as flaky during the
- # current RSpec run
- attr_reader :suite_flaky_examples, :flaky_examples
-
- def initialize(suite_flaky_examples_json = nil)
- @flaky_examples = FlakyExamplesCollection.new
- @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json)
- end
-
- def example_passed(notification)
- current_example = Example.new(notification.example)
-
- return unless current_example.attempts > 1
-
- flaky_example = suite_flaky_examples.fetch(current_example.uid) do
- FlakyExample.new(current_example.to_h)
- end
- flaky_example.update!(current_example.to_h)
-
- flaky_examples[current_example.uid] = flaky_example
- end
-
- def dump_summary(_)
- Report.new(flaky_examples).write(Config.flaky_examples_report_path)
-
- return unless new_flaky_examples.any?
-
- rails_logger_warn("\nNew flaky examples detected:\n")
- rails_logger_warn(JSON.pretty_generate(new_flaky_examples.to_h))
-
- Report.new(new_flaky_examples).write(Config.new_flaky_examples_report_path)
- end
-
- private
-
- def new_flaky_examples
- @new_flaky_examples ||= flaky_examples - suite_flaky_examples
- end
-
- def init_suite_flaky_examples(suite_flaky_examples_json = nil)
- if suite_flaky_examples_json
- Report.load_json(suite_flaky_examples_json).flaky_examples
- else
- return {} unless File.exist?(Config.suite_flaky_examples_report_path)
-
- Report.load(Config.suite_flaky_examples_report_path).flaky_examples
- end
- end
-
- def rails_logger_warn(text)
- target = defined?(Rails) ? Rails.logger : Kernel
-
- target.warn(text)
- end
- end
-end
diff --git a/gems/rspec_flaky/lib/rspec_flaky/report.rb b/gems/rspec_flaky/lib/rspec_flaky/report.rb
deleted file mode 100644
index cc213d336ae..00000000000
--- a/gems/rspec_flaky/lib/rspec_flaky/report.rb
+++ /dev/null
@@ -1,59 +0,0 @@
-# frozen_string_literal: true
-
-require 'json'
-require 'time'
-require 'fileutils'
-
-require_relative 'config'
-require_relative 'flaky_examples_collection'
-
-module RspecFlaky
- # This class is responsible for loading/saving JSON reports, and pruning
- # outdated examples.
- class Report < SimpleDelegator
- OUTDATED_DAYS_THRESHOLD = 7
-
- attr_reader :flaky_examples
-
- def self.load(file_path)
- load_json(File.read(file_path))
- end
-
- def self.load_json(json)
- new(FlakyExamplesCollection.new(JSON.parse(json)))
- end
-
- def initialize(flaky_examples)
- unless flaky_examples.is_a?(FlakyExamplesCollection)
- raise ArgumentError,
- "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, #{flaky_examples.class} given!"
- end
-
- @flaky_examples = flaky_examples
- super(flaky_examples)
- end
-
- def write(file_path)
- unless Config.generate_report?
- Kernel.warn "! Generating reports is disabled. To enable it, please set the `FLAKY_RSPEC_GENERATE_REPORT=1` !"
- return
- end
-
- report_path_dir = File.dirname(file_path)
- FileUtils.mkdir_p(report_path_dir)
-
- File.write(file_path, JSON.pretty_generate(flaky_examples.to_h))
- end
-
- def prune_outdated(days: OUTDATED_DAYS_THRESHOLD)
- outdated_date_threshold = Time.now - (3600 * 24 * days)
- recent_flaky_examples = flaky_examples.dup
- .delete_if do |_uid, flaky_example|
- last_flaky_at = flaky_example.to_h[:last_flaky_at]
- last_flaky_at && last_flaky_at.to_i < outdated_date_threshold.to_i
- end
-
- self.class.new(FlakyExamplesCollection.new(recent_flaky_examples))
- end
- end
-end