diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-02-10 09:17:41 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-02-10 09:17:41 +0300 |
commit | b4ea95860f6fa2621539940db6d3b4363f2c639b (patch) | |
tree | 6eaae2c497d2a4fd0008da48bb2a4c04c2de5e75 | |
parent | 8e6b1dc7f7dfb9a7f8431f7c246415e34d31870a (diff) |
Add latest changes from gitlab-org/gitlab@master
28 files changed, 290 insertions, 176 deletions
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 2647e38a793..702bba8ae97 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -54,7 +54,6 @@ - knapsack/ - rspec/ - tmp/capybara/ - - tmp/memory_test/ - log/*.log reports: junit: ${JUNIT_RESULT_FILE} @@ -312,6 +311,8 @@ rspec system pg12: - .rspec-base-pg12 - .rails:rules:ee-and-foss-system - .rspec-system-parallel + variables: + DEBUG_GITLAB_TRANSACTION_STACK: "true" rspec system pg12 minimal: extends: @@ -571,7 +572,6 @@ rspec:coverage: - rspec system pg12-as-if-foss decomposed script: - run_timed_command "bundle exec scripts/merge-simplecov" - - run_timed_command "bundle exec scripts/gather-test-memory-data" coverage: '/LOC \((\d+\.\d+%)\) covered.$/' artifacts: name: coverage @@ -580,7 +580,6 @@ rspec:coverage: - coverage/index.html - coverage/assets/ - coverage/lcov/ - - tmp/memory_test/ reports: cobertura: coverage/coverage.xml diff --git a/app/assets/javascripts/issues/list/components/issues_list_app.vue b/app/assets/javascripts/issues/list/components/issues_list_app.vue index fe09f840b28..958493cbebd 100644 --- a/app/assets/javascripts/issues/list/components/issues_list_app.vue +++ b/app/assets/javascripts/issues/list/components/issues_list_app.vue @@ -41,7 +41,6 @@ import { i18n, MAX_LIST_SIZE, PAGE_SIZE, - PARAM_DUE_DATE, PARAM_STATE, RELATIVE_POSITION_ASC, TOKEN_TYPE_ASSIGNEE, @@ -65,7 +64,6 @@ import { convertToApiParams, convertToSearchQuery, convertToUrlParams, - getDueDateValue, getFilterTokens, getInitialPageParams, getSortKey, @@ -161,7 +159,6 @@ export default { } return { - dueDateFilter: getDueDateValue(getParameterByName(PARAM_DUE_DATE)), exportCsvPathWithQuery: this.getExportCsvPathWithQuery(), filterTokens: isSearchDisabled ? [] : getFilterTokens(window.location.search), issues: [], @@ -391,7 +388,6 @@ export default { }, urlParams() { return { - due_date: this.dueDateFilter, search: this.searchQuery, sort: urlSortParams[this.sortKey], state: this.state, diff --git a/app/assets/javascripts/issues/list/constants.js b/app/assets/javascripts/issues/list/constants.js index 4a380848b4f..284167a933f 100644 --- a/app/assets/javascripts/issues/list/constants.js +++ b/app/assets/javascripts/issues/list/constants.js @@ -55,8 +55,6 @@ export const i18n = { export const MAX_LIST_SIZE = 10; export const PAGE_SIZE = 20; export const PAGE_SIZE_MANUAL = 100; -export const PARAM_DUE_DATE = 'due_date'; -export const PARAM_SORT = 'sort'; export const PARAM_STATE = 'state'; export const RELATIVE_POSITION = 'relative_position'; @@ -68,21 +66,6 @@ export const largePageSizeParams = { firstPageSize: PAGE_SIZE_MANUAL, }; -export const DUE_DATE_NONE = '0'; -export const DUE_DATE_ANY = ''; -export const DUE_DATE_OVERDUE = 'overdue'; -export const DUE_DATE_WEEK = 'week'; -export const DUE_DATE_MONTH = 'month'; -export const DUE_DATE_NEXT_MONTH_AND_PREVIOUS_TWO_WEEKS = 'next_month_and_previous_two_weeks'; -export const DUE_DATE_VALUES = [ - DUE_DATE_NONE, - DUE_DATE_ANY, - DUE_DATE_OVERDUE, - DUE_DATE_WEEK, - DUE_DATE_MONTH, - DUE_DATE_NEXT_MONTH_AND_PREVIOUS_TWO_WEEKS, -]; - export const BLOCKING_ISSUES_ASC = 'BLOCKING_ISSUES_ASC'; export const BLOCKING_ISSUES_DESC = 'BLOCKING_ISSUES_DESC'; export const CREATED_ASC = 'CREATED_ASC'; diff --git a/app/assets/javascripts/issues/list/utils.js b/app/assets/javascripts/issues/list/utils.js index 2bc21e57cb7..6322968b3f0 100644 --- a/app/assets/javascripts/issues/list/utils.js +++ b/app/assets/javascripts/issues/list/utils.js @@ -13,7 +13,6 @@ import { defaultPageSizeParams, DUE_DATE_ASC, DUE_DATE_DESC, - DUE_DATE_VALUES, filters, LABEL_PRIORITY_ASC, LABEL_PRIORITY_DESC, @@ -52,8 +51,6 @@ export const getSortKey = (sort) => export const isSortKey = (sort) => Object.keys(urlSortParams).includes(sort); -export const getDueDateValue = (value) => (DUE_DATE_VALUES.includes(value) ? value : undefined); - export const getSortOptions = (hasIssueWeightsFeature, hasBlockedIssuesFeature) => { const sortOptions = [ { diff --git a/app/finders/releases_finder.rb b/app/finders/releases_finder.rb index 0cfa4310ab7..0d72d6ffc6b 100644 --- a/app/finders/releases_finder.rb +++ b/app/finders/releases_finder.rb @@ -38,19 +38,17 @@ class ReleasesFinder if parent.is_a?(Project) Ability.allowed?(current_user, :read_release, parent) ? [parent] : [] elsif parent.is_a?(Group) - accessible_projects + Ability.allowed?(current_user, :read_release, parent) ? accessible_projects : [] end end end def accessible_projects - projects = if include_subgroups? - Project.for_group_and_its_subgroups(parent) - else - parent.projects - end - - projects.select { |project| Ability.allowed?(current_user, :read_release, project) } + if include_subgroups? + Project.for_group_and_its_subgroups(parent) + else + parent.projects + end end # rubocop: disable CodeReuse/ActiveRecord diff --git a/app/models/concerns/cross_database_modification.rb b/app/models/concerns/cross_database_modification.rb index 9efff6d580a..6ee3a463ed2 100644 --- a/app/models/concerns/cross_database_modification.rb +++ b/app/models/concerns/cross_database_modification.rb @@ -4,15 +4,45 @@ module CrossDatabaseModification extend ActiveSupport::Concern class TransactionStackTrackRecord + DEBUG_STACK = Rails.env.test? && ENV['DEBUG_GITLAB_TRANSACTION_STACK'] + LOG_FILENAME = Rails.root.join("log", "gitlab_transaction_stack.log") + + def self.logger + @logger ||= Logger.new(LOG_FILENAME, formatter: ->(_, _, _, msg) { Gitlab::Json.dump(msg) + "\n" }) + end + + def self.log_gitlab_transactions_stack(action: nil, example: nil) + return unless DEBUG_STACK + + message = "gitlab_transactions_stack performing #{action}" + message += " in example #{example}" if example + + cleaned_backtrace = Gitlab::BacktraceCleaner.clean_backtrace(caller) + .reject { |line| line.include?('lib/gitlab/database/query_analyzer') } + .first(5) + + logger.warn({ + message: message, + action: action, + gitlab_transactions_stack: ::ApplicationRecord.gitlab_transactions_stack, + caller: cleaned_backtrace, + thread: Thread.current.object_id + }) + end + def initialize(subject, gitlab_schema) @subject = subject @gitlab_schema = gitlab_schema @subject.gitlab_transactions_stack.push(gitlab_schema) + + self.class.log_gitlab_transactions_stack(action: :after_push) end def done! unless @done @done = true + + self.class.log_gitlab_transactions_stack(action: :before_pop) @subject.gitlab_transactions_stack.pop end diff --git a/app/models/namespace_statistics.rb b/app/models/namespace_statistics.rb index 9a6244dbde8..04ca05d85ff 100644 --- a/app/models/namespace_statistics.rb +++ b/app/models/namespace_statistics.rb @@ -29,6 +29,10 @@ class NamespaceStatistics < ApplicationRecord # rubocop:disable Gitlab/Namespace end def update_storage_size + # This prevents failures with older database schemas, such as those + # in migration specs. + return unless self.class.database.cached_column_exists?(:dependency_proxy_size) + self.storage_size = dependency_proxy_size end diff --git a/app/policies/group_policy.rb b/app/policies/group_policy.rb index 62111c55e83..76e5b3ece53 100644 --- a/app/policies/group_policy.rb +++ b/app/policies/group_policy.rb @@ -100,6 +100,7 @@ class GroupPolicy < Namespaces::GroupProjectNamespaceSharedPolicy enable :read_group enable :upload_file enable :guest_access + enable :read_release end rule { admin }.policy do diff --git a/db/post_migrate/20220120123700_add_tmp_index_routes_id_for_namespaces.rb b/db/post_migrate/20220120123700_add_tmp_index_routes_id_for_namespaces.rb new file mode 100644 index 00000000000..2b8cecd0e57 --- /dev/null +++ b/db/post_migrate/20220120123700_add_tmp_index_routes_id_for_namespaces.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddTmpIndexRoutesIdForNamespaces < Gitlab::Database::Migration[1.0] + INDEX_NAME = 'tmp_index_for_namespace_id_migration_on_routes' + + disable_ddl_transaction! + + def up + # Temporary index to be removed in 14.9 + # https://gitlab.com/gitlab-org/gitlab/-/issues/352353 + add_concurrent_index :routes, :id, where: "routes.namespace_id is null and routes.source_type = 'Namespace'", name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :routes, INDEX_NAME + end +end diff --git a/db/post_migrate/20220120123800_backfill_namespace_id_for_namespace_routes.rb b/db/post_migrate/20220120123800_backfill_namespace_id_for_namespace_routes.rb new file mode 100644 index 00000000000..4aac0903502 --- /dev/null +++ b/db/post_migrate/20220120123800_backfill_namespace_id_for_namespace_routes.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class BackfillNamespaceIdForNamespaceRoutes < Gitlab::Database::Migration[1.0] + MIGRATION = 'BackfillNamespaceIdForNamespaceRoute' + INTERVAL = 2.minutes + BATCH_SIZE = 1_000 + MAX_BATCH_SIZE = 10_000 + SUB_BATCH_SIZE = 200 + + def up + queue_batched_background_migration( + MIGRATION, + :routes, + :id, + job_interval: INTERVAL, + batch_size: BATCH_SIZE, + max_batch_size: MAX_BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + Gitlab::Database::BackgroundMigration::BatchedMigration + .for_configuration(MIGRATION, :routes, :id, []) + .delete_all + end +end diff --git a/db/schema_migrations/20220120123700 b/db/schema_migrations/20220120123700 new file mode 100644 index 00000000000..263901fff19 --- /dev/null +++ b/db/schema_migrations/20220120123700 @@ -0,0 +1 @@ +9e274eae18520821dd890a11c8c6192da82a6051dce9ec2934b1365e57a10fdb
\ No newline at end of file diff --git a/db/schema_migrations/20220120123800 b/db/schema_migrations/20220120123800 new file mode 100644 index 00000000000..6d271f75c08 --- /dev/null +++ b/db/schema_migrations/20220120123800 @@ -0,0 +1 @@ +9d2f0b0d2cb1a5844bdca5bcb4b82fa5cc6b465fa19177f9f6ca16574128fdc8
\ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 23832e79e19..80dec806173 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -28311,6 +28311,8 @@ CREATE INDEX tmp_idx_deduplicate_vulnerability_occurrences ON vulnerability_occu CREATE INDEX tmp_idx_vulnerability_occurrences_on_id_where_report_type_7_99 ON vulnerability_occurrences USING btree (id) WHERE (report_type = ANY (ARRAY[7, 99])); +CREATE INDEX tmp_index_for_namespace_id_migration_on_routes ON routes USING btree (id) WHERE ((namespace_id IS NULL) AND ((source_type)::text = 'Namespace'::text)); + CREATE INDEX tmp_index_members_on_state ON members USING btree (state) WHERE (state = 2); CREATE INDEX tmp_index_namespaces_empty_traversal_ids_with_child_namespaces ON namespaces USING btree (id) WHERE ((parent_id IS NOT NULL) AND (traversal_ids = '{}'::integer[])); diff --git a/doc/ci/yaml/index.md b/doc/ci/yaml/index.md index 99e4536e94f..d9ea5498b63 100644 --- a/doc/ci/yaml/index.md +++ b/doc/ci/yaml/index.md @@ -974,7 +974,7 @@ failure. - `on_success` (default): Upload artifacts only when the job succeeds. - `on_failure`: Upload artifacts only when the job fails. -- `always`: Always upload artifacts. For example, when +- `always`: Always upload artifacts (except when jobs time out). For example, when [uploading artifacts](../unit_test_reports.md#viewing-junit-screenshots-on-gitlab) required to troubleshoot failing tests. diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 0b3b35f135a..91dd02a9dd5 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -570,6 +570,32 @@ The following variables are used for configuring specific analyzers (used for a | `RETIREJS_NODE_ADVISORY_DB` | `retire.js` | `https://raw.githubusercontent.com/RetireJS/retire.js/master/repository/npmrepository.json` | Path or URL to `retire.js` node vulnerability data file. Note that if the URL hosting the data file uses a custom SSL certificate, for example in an offline installation, you can pass the certificate in the `ADDITIONAL_CA_CERT_BUNDLE` variable. | | `RETIREJS_ADVISORY_DB_INSECURE` | `retire.js` | `false` | Enable fetching remote JS and Node vulnerability data files (defined by the `RETIREJS_JS_ADVISORY_DB` and `RETIREJS_NODE_ADVISORY_DB` variables) from hosts using an insecure or self-signed SSL (TLS) certificate. | +#### Other variables + +The previous tables are not an exhaustive list of all variables that can be used. They contain all specific GitLab and analyzer variables we support and test. There are many variables, such as environment variables, that you can pass in and they will work. This is a large list, many of which we may be unaware of, and as such is not documented. + +For example, to pass the non-GitLab environment variable `HTTPS_PROXY` to all Dependency Scanning jobs, +set it as a [custom CI/CD variable in your `.gitlab-ci.yml`](../../../ci/variables/#create-a-custom-cicd-variable-in-the-gitlab-ciyml-file) +file like this: + +```yaml +variables: + HTTPS_PROXY: "https://squid-proxy:3128" +``` + +Alternatively we may use it in specific jobs, like Dependency Scanning: + +```yaml +dependency_scanning: + variables: + HTTPS_PROXY: $HTTPS_PROXY +``` + +As we have not tested all variables you may find some will work and others will not. +If one does not work and you need it we suggest +[submitting a feature request](https://gitlab.com/gitlab-org/gitlab/-/issues/new?issuable_template=Feature%20proposal%20-%20detailed&issue[title]=Docs%20feedback%20-%20feature%20proposal:%20Write%20your%20title) +or [contributing to the code](../../../development/index.md) to enable it to be used. + ### Using a custom SSL CA certificate authority You can use the `ADDITIONAL_CA_CERT_BUNDLE` CI/CD variable to configure a custom SSL CA certificate authority. The `ADDITIONAL_CA_CERT_BUNDLE` value should contain the [text representation of the X.509 PEM public-key certificate](https://tools.ietf.org/html/rfc7468#section-5.1). For example, to configure this value in the `.gitlab-ci.yml` file, use the following: diff --git a/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb b/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb new file mode 100644 index 00000000000..fe3edd3322b --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the `routes.namespace_id` column, by copying source_id value + # (for groups and user namespaces source_id == namespace_id) + class BackfillNamespaceIdForNamespaceRoute + include Gitlab::Database::DynamicModelHelpers + + def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) + parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) + + parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + batch_metrics.time_operation(:update_all) do + sub_batch.update_all('namespace_id=source_id') + end + + pause_ms = [0, pause_ms].max + sleep(pause_ms * 0.001) + end + end + + def batch_metrics + @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new + end + + private + + def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) + define_batchable_model(source_table, connection: ActiveRecord::Base.connection) + .joins('inner join namespaces on routes.source_id = namespaces.id') + .where(source_key_column => start_id..stop_id) + .where(namespace_id: nil) + .where(source_type: 'Namespace') + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index 1dcbd86f1cd..a604f79dc41 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -56,6 +56,9 @@ module Gitlab context[:transaction_depth_by_db][database] -= 1 if context[:transaction_depth_by_db][database] == 0 context[:modified_tables_by_db][database].clear + + # Attempt to troubleshoot https://gitlab.com/gitlab-org/gitlab/-/issues/351531 + ::CrossDatabaseModification::TransactionStackTrackRecord.log_gitlab_transactions_stack(action: :end_of_transaction) elsif context[:transaction_depth_by_db][database] < 0 context[:transaction_depth_by_db][database] = 0 raise CrossDatabaseModificationAcrossUnsupportedTablesError, "Misaligned cross-DB transactions discovered at query #{sql}. This could be a bug in #{self.class} or a valid issue to investigate. Read more at https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions ." diff --git a/scripts/gather-test-memory-data b/scripts/gather-test-memory-data deleted file mode 100755 index 3156365ac19..00000000000 --- a/scripts/gather-test-memory-data +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'csv' - -def join_csv_files(output_path, input_paths) - return if input_paths.empty? - - input_csvs = input_paths.map do |input_path| - CSV.read(input_path, headers: true) - end - - CSV.open(output_path, "w", headers: input_csvs.first.headers, write_headers: true) do |output_csv| - input_csvs.each do |input_csv| - input_csv.each do |line| - output_csv << line - end - end - end -end - -join_csv_files('tmp/memory_test/report.csv', Dir['tmp/memory_test/*.csv'].sort) diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh index 3b1112725e3..9fbcf0e5fd7 100644 --- a/scripts/rspec_helpers.sh +++ b/scripts/rspec_helpers.sh @@ -268,10 +268,6 @@ function rspec_paralellized_job() { debug_rspec_variables - mkdir -p tmp/memory_test - - export MEMORY_TEST_PATH="tmp/memory_test/${report_name}_memory.csv" - if [[ -n $RSPEC_TESTS_MAPPING_ENABLED ]]; then tooling/bin/parallel_rspec --rspec_args "$(rspec_args "${rspec_opts}")" --filter "tmp/matching_tests.txt" || rspec_run_status=$? else diff --git a/spec/controllers/groups/releases_controller_spec.rb b/spec/controllers/groups/releases_controller_spec.rb index 50701382945..582a77b1c50 100644 --- a/spec/controllers/groups/releases_controller_spec.rb +++ b/spec/controllers/groups/releases_controller_spec.rb @@ -6,14 +6,14 @@ RSpec.describe Groups::ReleasesController do let(:group) { create(:group) } let!(:project) { create(:project, :repository, :public, namespace: group) } let!(:private_project) { create(:project, :repository, :private, namespace: group) } - let(:developer) { create(:user) } + let(:guest) { create(:user) } let!(:release_1) { create(:release, project: project, tag: 'v1', released_at: Time.zone.parse('2020-02-15')) } let!(:release_2) { create(:release, project: project, tag: 'v2', released_at: Time.zone.parse('2020-02-20')) } let!(:private_release_1) { create(:release, project: private_project, tag: 'p1', released_at: Time.zone.parse('2020-03-01')) } let!(:private_release_2) { create(:release, project: private_project, tag: 'p2', released_at: Time.zone.parse('2020-03-05')) } before do - private_project.add_developer(developer) + group.add_guest(guest) end describe 'GET #index' do @@ -42,7 +42,7 @@ RSpec.describe Groups::ReleasesController do end it 'does not return any releases' do - expect(json_response.map {|r| r['tag'] } ).to match_array(%w(v2 v1)) + expect(json_response.map {|r| r['tag'] } ).to be_empty end it 'returns OK' do @@ -52,7 +52,7 @@ RSpec.describe Groups::ReleasesController do context 'the user is authorized' do it "returns all group's public and private project's releases as JSON, ordered by released_at" do - sign_in(developer) + sign_in(guest) subject diff --git a/spec/finders/releases_finder_spec.rb b/spec/finders/releases_finder_spec.rb index 94b6fe53daa..5ddb5c33fad 100644 --- a/spec/finders/releases_finder_spec.rb +++ b/spec/finders/releases_finder_spec.rb @@ -23,6 +23,16 @@ RSpec.describe ReleasesFinder do end end + shared_examples_for 'when the user is not part of the group' do + before do + allow(Ability).to receive(:allowed?).with(user, :read_release, group).and_return(false) + end + + it 'returns no releases' do + is_expected.to be_empty + end + end + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/27716 shared_examples_for 'when tag is nil' do before do @@ -66,9 +76,9 @@ RSpec.describe ReleasesFinder do it_behaves_like 'when the user is not part of the project' - context 'when the user is a project developer' do + context 'when the user is a project guest' do before do - project.add_developer(user) + project.add_guest(user) end it 'sorts by release date' do @@ -118,25 +128,24 @@ RSpec.describe ReleasesFinder do subject { described_class.new(group, user, params).execute(**args) } - it_behaves_like 'when the user is not part of the project' + it_behaves_like 'when the user is not part of the group' - context 'when the user is a project developer on one sibling project' do + context 'when the user is a project guest on one sibling project' do before do - project.add_developer(user) + project.add_guest(user) v1_0_0.update_attribute(:released_at, 3.days.ago) v1_1_0.update_attribute(:released_at, 1.day.ago) end - it 'sorts by release date' do - expect(subject.size).to eq(2) - expect(subject).to eq([v1_1_0, v1_0_0]) + it 'does not return any releases' do + expect(subject.size).to eq(0) + expect(subject).to eq([]) end end - context 'when the user is a project developer on all projects' do + context 'when the user is a guest on the group' do before do - project.add_developer(user) - project2.add_developer(user) + group.add_guest(user) v1_0_0.update_attribute(:released_at, 3.days.ago) v6.update_attribute(:released_at, 2.days.ago) v1_1_0.update_attribute(:released_at, 1.day.ago) @@ -161,22 +170,21 @@ RSpec.describe ReleasesFinder do let(:project2) { create(:project, :repository, namespace: subgroup) } let!(:v6) { create(:release, project: project2, tag: 'v6') } - it_behaves_like 'when the user is not part of the project' + it_behaves_like 'when the user is not part of the group' - context 'when the user a project developer in the subgroup project' do + context 'when the user a project guest in the subgroup project' do before do - project2.add_developer(user) + project2.add_guest(user) end - it 'returns only the subgroup releases' do - expect(subject).to match_array([v6]) + it 'does not return any releases' do + expect(subject).to match_array([]) end end - context 'when the user a project developer in both projects' do + context 'when the user is a guest on the group' do before do - project.add_developer(user) - project2.add_developer(user) + group.add_guest(user) v6.update_attribute(:released_at, 2.days.ago) end @@ -201,34 +209,32 @@ RSpec.describe ReleasesFinder do p3.update_attribute(:released_at, 3.days.ago) end - it_behaves_like 'when the user is not part of the project' + it_behaves_like 'when the user is not part of the group' - context 'when the user a project developer in the subgroup and subsubgroup project' do + context 'when the user a project guest in the subgroup and subsubgroup project' do before do - project2.add_developer(user) - project3.add_developer(user) + project2.add_guest(user) + project3.add_guest(user) end - it 'returns only the subgroup and subsubgroup releases' do - expect(subject).to match_array([v6, p3]) + it 'does not return any releases' do + expect(subject).to match_array([]) end end - context 'when the user a project developer in the subsubgroup project' do + context 'when the user a project guest in the subsubgroup project' do before do - project3.add_developer(user) + project3.add_guest(user) end - it 'returns only the subsubgroup releases' do - expect(subject).to match_array([p3]) + it 'does not return any releases' do + expect(subject).to match_array([]) end end - context 'when the user a project developer in all projects' do + context 'when the user a guest on the group' do before do - project.add_developer(user) - project2.add_developer(user) - project3.add_developer(user) + group.add_guest(user) end it 'returns all releases' do diff --git a/spec/frontend/issues/list/components/issues_list_app_spec.js b/spec/frontend/issues/list/components/issues_list_app_spec.js index 1f2b3ad4cb1..b7fb5bc7e61 100644 --- a/spec/frontend/issues/list/components/issues_list_app_spec.js +++ b/spec/frontend/issues/list/components/issues_list_app_spec.js @@ -30,8 +30,6 @@ import IssuesListApp from '~/issues/list/components/issues_list_app.vue'; import NewIssueDropdown from '~/issues/list/components/new_issue_dropdown.vue'; import { CREATED_DESC, - DUE_DATE_OVERDUE, - PARAM_DUE_DATE, RELATIVE_POSITION, RELATIVE_POSITION_ASC, TOKEN_TYPE_ASSIGNEE, @@ -296,16 +294,6 @@ describe('CE IssuesListApp component', () => { }); describe('initial url params', () => { - describe('due_date', () => { - it('is set from the url params', () => { - setWindowLocation(`?${PARAM_DUE_DATE}=${DUE_DATE_OVERDUE}`); - - wrapper = mountComponent(); - - expect(findIssuableList().props('urlParams')).toMatchObject({ due_date: DUE_DATE_OVERDUE }); - }); - }); - describe('search', () => { it('is set from the url params', () => { setWindowLocation(locationSearch); diff --git a/spec/frontend/issues/list/utils_spec.js b/spec/frontend/issues/list/utils_spec.js index 487625bdc7c..1d3e94df897 100644 --- a/spec/frontend/issues/list/utils_spec.js +++ b/spec/frontend/issues/list/utils_spec.js @@ -10,7 +10,6 @@ import { } from 'jest/issues/list/mock_data'; import { defaultPageSizeParams, - DUE_DATE_VALUES, largePageSizeParams, RELATIVE_POSITION_ASC, urlSortParams, @@ -19,7 +18,6 @@ import { convertToApiParams, convertToSearchQuery, convertToUrlParams, - getDueDateValue, getFilterTokens, getInitialPageParams, getSortKey, @@ -56,16 +54,6 @@ describe('isSortKey', () => { }); }); -describe('getDueDateValue', () => { - it.each(DUE_DATE_VALUES)('returns the argument when it is `%s`', (value) => { - expect(getDueDateValue(value)).toBe(value); - }); - - it('returns undefined when the argument is invalid', () => { - expect(getDueDateValue('invalid value')).toBeUndefined(); - }); -}); - describe('getSortOptions', () => { describe.each` hasIssueWeightsFeature | hasBlockedIssuesFeature | length | containsWeight | containsBlocking diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb new file mode 100644 index 00000000000..b821efcadb0 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_namespace_id_for_namespace_route_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdForNamespaceRoute, :migration, schema: 20220120123800 do + let(:migration) { described_class.new } + let(:namespaces_table) { table(:namespaces) } + let(:projects_table) { table(:projects) } + let(:routes_table) { table(:routes) } + + let(:table_name) { 'routes' } + let(:batch_column) { :id } + let(:sub_batch_size) { 200 } + let(:pause_ms) { 0 } + + let(:namespace1) { namespaces_table.create!(name: 'namespace1', path: 'namespace1', type: 'User') } + let(:namespace2) { namespaces_table.create!(name: 'namespace2', path: 'namespace2', type: 'Group') } + let(:namespace3) { namespaces_table.create!(name: 'namespace3', path: 'namespace3', type: 'Group') } + let(:namespace4) { namespaces_table.create!(name: 'namespace4', path: 'namespace4', type: 'Group') } + let(:project1) { projects_table.create!(name: 'project1', namespace_id: namespace1.id) } + + subject(:perform_migration) { migration.perform(1, 10, table_name, batch_column, sub_batch_size, pause_ms) } + + before do + routes_table.create!(id: 1, name: 'test1', path: 'test1', source_id: namespace1.id, + source_type: namespace1.class.sti_name) + routes_table.create!(id: 2, name: 'test2', path: 'test2', source_id: namespace2.id, + source_type: namespace2.class.sti_name) + routes_table.create!(id: 5, name: 'test3', path: 'test3', source_id: project1.id, + source_type: project1.class.sti_name) # should be ignored - project route + routes_table.create!(id: 6, name: 'test4', path: 'test4', source_id: non_existing_record_id, + source_type: namespace3.class.sti_name) # should be ignored - invalid source_id + routes_table.create!(id: 10, name: 'test5', path: 'test5', source_id: namespace3.id, + source_type: namespace3.class.sti_name) + routes_table.create!(id: 11, name: 'test6', path: 'test6', source_id: namespace4.id, + source_type: namespace4.class.sti_name) # should be ignored - outside the scope + end + + it 'backfills `type` for the selected records', :aggregate_failures do + perform_migration + + expect(routes_table.where.not(namespace_id: nil).pluck(:id)).to match_array([1, 2, 10]) + end + + it 'tracks timings of queries' do + expect(migration.batch_metrics.timings).to be_empty + + expect { perform_migration }.to change { migration.batch_metrics.timings } + end +end diff --git a/spec/migrations/backfill_namespace_id_for_namespace_routes_spec.rb b/spec/migrations/backfill_namespace_id_for_namespace_routes_spec.rb new file mode 100644 index 00000000000..913ec404795 --- /dev/null +++ b/spec/migrations/backfill_namespace_id_for_namespace_routes_spec.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe BackfillNamespaceIdForNamespaceRoutes do + let_it_be(:migration) { described_class::MIGRATION } + + describe '#up' do + it 'schedules background jobs for each batch of routes' do + migrate! + + expect(migration).to have_scheduled_batched_migration( + table_name: :routes, + column_name: :id, + interval: described_class::INTERVAL + ) + end + end + + describe '#down' do + it 'deletes all batched migration records' do + migrate! + schema_migrate_down! + + expect(migration).not_to have_scheduled_batched_migration + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 2b3b7b114c8..f823683c274 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -114,6 +114,15 @@ RSpec.configure do |config| config.run_all_when_everything_filtered = true end + # Attempt to troubleshoot https://gitlab.com/gitlab-org/gitlab/-/issues/351531 + config.after do |example| + if example.exception.is_a?(Gitlab::Database::QueryAnalyzers::PreventCrossDatabaseModification::CrossDatabaseModificationAcrossUnsupportedTablesError) + ::CrossDatabaseModification::TransactionStackTrackRecord.log_gitlab_transactions_stack(action: :after_failure, example: example.description) + else + ::CrossDatabaseModification::TransactionStackTrackRecord.log_gitlab_transactions_stack(action: :after_example, example: example.description) + end + end + # Re-run failures locally with `--only-failures` config.example_status_persistence_file_path = ENV.fetch('RSPEC_LAST_RUN_RESULTS_FILE', './spec/examples.txt') @@ -184,7 +193,6 @@ RSpec.configure do |config| config.include RedisHelpers config.include Rails.application.routes.url_helpers, type: :routing config.include PolicyHelpers, type: :policy - config.include MemoryUsageHelper config.include ExpectRequestWithStatus, type: :request config.include IdempotentWorkerHelper, type: :worker config.include RailsHelpers @@ -244,10 +252,6 @@ RSpec.configure do |config| ::Ci::ApplicationRecord.set_open_transactions_baseline end - config.append_before do - Thread.current[:current_example_group] = ::RSpec.current_example.metadata[:example_group] - end - config.append_after do ApplicationRecord.reset_open_transactions_baseline ::Ci::ApplicationRecord.reset_open_transactions_baseline diff --git a/spec/support/helpers/memory_usage_helper.rb b/spec/support/helpers/memory_usage_helper.rb deleted file mode 100644 index 02d1935921f..00000000000 --- a/spec/support/helpers/memory_usage_helper.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module MemoryUsageHelper - extend ActiveSupport::Concern - - def gather_memory_data(csv_path) - write_csv_entry(csv_path, - { - example_group_path: TestEnv.topmost_example_group[:location], - example_group_description: TestEnv.topmost_example_group[:description], - time: Time.current, - job_name: ENV['CI_JOB_NAME'] - }.merge(get_memory_usage)) - end - - def write_csv_entry(path, entry) - CSV.open(path, "a", headers: entry.keys, write_headers: !File.exist?(path)) do |file| - file << entry.values - end - end - - def get_memory_usage - output, status = Gitlab::Popen.popen(%w(free -m)) - abort "`free -m` return code is #{status}: #{output}" unless status == 0 - - result = output.split("\n")[1].split(" ")[1..] - attrs = %i(m_total m_used m_free m_shared m_buffers_cache m_available).freeze - - attrs.zip(result).to_h - end - - included do |config| - config.after(:all) do - gather_memory_data(ENV['MEMORY_TEST_PATH']) if ENV['MEMORY_TEST_PATH'] - end - end -end diff --git a/spec/support/helpers/test_env.rb b/spec/support/helpers/test_env.rb index 5c3ca92c4d0..18c25f4b770 100644 --- a/spec/support/helpers/test_env.rb +++ b/spec/support/helpers/test_env.rb @@ -371,17 +371,6 @@ module TestEnv FileUtils.rm_rf(path) end - def current_example_group - Thread.current[:current_example_group] - end - - # looking for a top-level `describe` - def topmost_example_group - example_group = current_example_group - example_group = example_group[:parent_example_group] until example_group[:parent_example_group].nil? - example_group - end - def seed_db Gitlab::DatabaseImporters::WorkItems::BaseTypeImporter.import end |