From 6c2b987064064500b42da924d86d43473bfd2b7f Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Tue, 19 Dec 2023 06:10:05 +0000 Subject: Add latest changes from gitlab-org/gitlab@master --- ...alize_system_note_metadata_bigint_conversion.rb | 25 ++ ...ing_milestone_id_column_from_vulnerabilities.rb | 24 ++ db/schema_migrations/20231204042048 | 1 + db/schema_migrations/20231217053910 | 1 + db/structure.sql | 6 - doc/development/pipelines/index.md | 3 + doc/tutorials/protected_workflow/index.md | 6 +- doc/user/discussions/index.md | 2 +- doc/user/okrs.md | 8 +- doc/user/project/issues/managing_issues.md | 4 +- doc/user/project/milestones/index.md | 4 +- .../custom_domains_ssl_tls_certification/index.md | 6 +- doc/user/project/repository/branches/index.md | 2 +- doc/user/tasks.md | 6 +- qa/qa/specs/features/sanity/feature_flags_spec.rb | 89 ---- qa/qa/specs/features/sanity/framework_spec.rb | 21 - qa/qa/specs/features/sanity/interception_spec.rb | 41 -- qa/qa/specs/features/sanity/version_spec.rb | 39 -- qa/qa/tools/reliable_report.rb | 102 ++++- qa/spec/tools/reliable_report_spec.rb | 478 +++++++++++++-------- 20 files changed, 462 insertions(+), 406 deletions(-) create mode 100644 db/post_migrate/20231204042048_finalize_system_note_metadata_bigint_conversion.rb create mode 100644 db/post_migrate/20231217053910_remove_due_date_sourcing_milestone_id_column_from_vulnerabilities.rb create mode 100644 db/schema_migrations/20231204042048 create mode 100644 db/schema_migrations/20231217053910 delete mode 100644 qa/qa/specs/features/sanity/feature_flags_spec.rb delete mode 100644 qa/qa/specs/features/sanity/framework_spec.rb delete mode 100644 qa/qa/specs/features/sanity/interception_spec.rb delete mode 100644 qa/qa/specs/features/sanity/version_spec.rb diff --git a/db/post_migrate/20231204042048_finalize_system_note_metadata_bigint_conversion.rb b/db/post_migrate/20231204042048_finalize_system_note_metadata_bigint_conversion.rb new file mode 100644 index 00000000000..2238c4e34c5 --- /dev/null +++ b/db/post_migrate/20231204042048_finalize_system_note_metadata_bigint_conversion.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class FinalizeSystemNoteMetadataBigintConversion < Gitlab::Database::Migration[2.2] + include Gitlab::Database::MigrationHelpers::ConvertToBigint + + disable_ddl_transaction! + restrict_gitlab_migration gitlab_schema: :gitlab_main + + milestone '16.7' + + TABLE_NAME = :system_note_metadata + + def up + ensure_batched_background_migration_is_finished( + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: TABLE_NAME, + column_name: 'id', + job_arguments: [['id'], ['id_convert_to_bigint']] + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20231217053910_remove_due_date_sourcing_milestone_id_column_from_vulnerabilities.rb b/db/post_migrate/20231217053910_remove_due_date_sourcing_milestone_id_column_from_vulnerabilities.rb new file mode 100644 index 00000000000..5cd41dff0ef --- /dev/null +++ b/db/post_migrate/20231217053910_remove_due_date_sourcing_milestone_id_column_from_vulnerabilities.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class RemoveDueDateSourcingMilestoneIdColumnFromVulnerabilities < Gitlab::Database::Migration[2.2] + disable_ddl_transaction! + + milestone '16.8' + + def up + with_lock_retries do + remove_column :vulnerabilities, :due_date_sourcing_milestone_id + end + end + + def down + unless column_exists?(:vulnerabilities, :due_date_sourcing_milestone_id) + add_column :vulnerabilities, :due_date_sourcing_milestone_id, :bigint + end + + # Add back index and constraint that were dropped in `up` + add_concurrent_index(:vulnerabilities, :due_date_sourcing_milestone_id) + add_concurrent_foreign_key(:vulnerabilities, :milestones, column: :due_date_sourcing_milestone_id, + on_delete: :nullify) + end +end diff --git a/db/schema_migrations/20231204042048 b/db/schema_migrations/20231204042048 new file mode 100644 index 00000000000..6e0e069539c --- /dev/null +++ b/db/schema_migrations/20231204042048 @@ -0,0 +1 @@ +ca81769223a50ac334a06e150a2454800292a0affa65e3b55f6925c0c1bc7947 \ No newline at end of file diff --git a/db/schema_migrations/20231217053910 b/db/schema_migrations/20231217053910 new file mode 100644 index 00000000000..ce8d6436f44 --- /dev/null +++ b/db/schema_migrations/20231217053910 @@ -0,0 +1 @@ +bf5f9ca0584e043c39ca57a9664241900c35cda921c94e0df2728b435137a066 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index b429b3ab4f1..3ed12685706 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -24904,7 +24904,6 @@ CREATE TABLE vulnerabilities ( description text, description_html text, start_date_sourcing_milestone_id bigint, - due_date_sourcing_milestone_id bigint, state smallint DEFAULT 1 NOT NULL, severity smallint NOT NULL, severity_overridden boolean DEFAULT false, @@ -35092,8 +35091,6 @@ CREATE INDEX index_vulnerabilities_on_detected_at_and_id ON vulnerabilities USIN CREATE INDEX index_vulnerabilities_on_dismissed_by_id ON vulnerabilities USING btree (dismissed_by_id); -CREATE INDEX index_vulnerabilities_on_due_date_sourcing_milestone_id ON vulnerabilities USING btree (due_date_sourcing_milestone_id); - CREATE INDEX index_vulnerabilities_on_epic_id ON vulnerabilities USING btree (epic_id); CREATE INDEX index_vulnerabilities_on_finding_id ON vulnerabilities USING btree (finding_id); @@ -37794,9 +37791,6 @@ ALTER TABLE ONLY catalog_resource_versions ALTER TABLE ONLY issue_customer_relations_contacts ADD CONSTRAINT fk_7b92f835bb FOREIGN KEY (contact_id) REFERENCES customer_relations_contacts(id) ON DELETE CASCADE; -ALTER TABLE ONLY vulnerabilities - ADD CONSTRAINT fk_7c5bb22a22 FOREIGN KEY (due_date_sourcing_milestone_id) REFERENCES milestones(id) ON DELETE SET NULL; - ALTER TABLE ONLY ssh_signatures ADD CONSTRAINT fk_7d2f93996c FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; diff --git a/doc/development/pipelines/index.md b/doc/development/pipelines/index.md index c4dfda9466a..cdcaae6a35e 100644 --- a/doc/development/pipelines/index.md +++ b/doc/development/pipelines/index.md @@ -268,6 +268,9 @@ the specific list of rules. If you want to force `e2e:package-and-test` to run regardless of your changes, you can add the `pipeline:run-all-e2e` label to the merge request. +The [`e2e:test-on-gdk`](../testing_guide/end_to_end/index.md#using-the-test-on-gdk-job) child pipeline runs `:reliable` +E2E specs automatically for all `code patterns changes`. See `.qa:rules:e2e-blocking` [`rules.gitlab-ci.yml`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/.gitlab/ci/rules.gitlab-ci.yml) for specific set of rules. + Consult the [End-to-end Testing](../testing_guide/end_to_end/index.md) dedicated page for more information. ### Review app jobs diff --git a/doc/tutorials/protected_workflow/index.md b/doc/tutorials/protected_workflow/index.md index 8e9ed3e952a..e9e3739d836 100644 --- a/doc/tutorials/protected_workflow/index.md +++ b/doc/tutorials/protected_workflow/index.md @@ -77,8 +77,8 @@ Next, add the subgroup as a member of the `engineering` group: 1. On the left sidebar, select **Search or go to** and search for `engineering`. Select the group named `Engineering`. -1. On the left sidebar, select **Manage > Members**. -1. On the top right, select **Invite a group**. +1. Select **Manage > Members**. +1. In the upper right, select **Invite a group**. 1. For **Select a group to invite**, select `Engineering / Managers`. 1. When adding the subgroups select the role **Maintainer**. This configures the highest role a member of the subgroup can inherit when accessing the `engineering` group and its projects. @@ -270,7 +270,7 @@ Your rules are now in place, even though no `1.*` branches exist yet: Now that all branch protections in place, you're ready to create your 1.0.0 release branch: 1. On the left sidebar, select **Code > Branches**. -1. On the top right, select **New branch**. Name it `1.0.0`. +1. In the upper-right corner, select **New branch**. Name it `1.0.0`. 1. Select **Create branch**. The branch protections are now visible in the UI: diff --git a/doc/user/discussions/index.md b/doc/user/discussions/index.md index 034e2e45127..81f15227c8c 100644 --- a/doc/user/discussions/index.md +++ b/doc/user/discussions/index.md @@ -163,7 +163,7 @@ To lock an issue or merge request: 1. On the left sidebar, select **Search or go to** and find your project. 1. For merge requests, select **Code > Merge requests**, and find your merge request. 1. For issues, select **Plan > Issues**, and find your issue. -1. On the top right, select **Merge request actions** or **Issue actions** (**{ellipsis_v}**), then select **Lock discussion**. +1. On the upper-right corner, select **Merge request actions** or **Issue actions** (**{ellipsis_v}**), then select **Lock discussion**. A system note is added to the page details. diff --git a/doc/user/okrs.md b/doc/user/okrs.md index 14e887fe297..0d2c311d1ef 100644 --- a/doc/user/okrs.md +++ b/doc/user/okrs.md @@ -220,7 +220,7 @@ Prerequisites: To promote a key result: 1. [Open the key result](#view-a-key-result). -1. In the top right corner, select the vertical ellipsis (**{ellipsis_v}**).. +1. In the upper-right corner, select the vertical ellipsis (**{ellipsis_v}**). 1. Select **Promote to objective**. Alternatively, use the `/promote_to objective` [quick action](../user/project/quick_actions.md). @@ -236,7 +236,7 @@ To copy the objective or key result reference to your clipboard: 1. On the left sidebar, select **Search or go to** and find your project. 1. Select **Plan > Issues**, then select your objective or key result to view it. -1. In the top right corner, select the vertical ellipsis (**{ellipsis_v}**), then select **Copy Reference**. +1. In the upper-right corner, select the vertical ellipsis (**{ellipsis_v}**), then select **Copy Reference**. You can now paste the reference into another description or comment. @@ -256,7 +256,7 @@ To copy the objective's or key result's email address: 1. On the left sidebar, select **Search or go to** and find your project. 1. Select **Plan > Issues**, then select your issue to view it. -1. In the top right corner, select the vertical ellipsis (**{ellipsis_v}**), then select **Copy objective email address** or **Copy key result email address**. +1. In the upper-right corner, select the vertical ellipsis (**{ellipsis_v}**), then select **Copy objective email address** or **Copy key result email address**. ## Close an OKR @@ -443,7 +443,7 @@ Prerequisites: To change the confidentiality of an existing OKR: 1. [Open the objective](#view-an-objective) or [key result](#view-a-key-result). -1. In the top right corner, select the vertical ellipsis (**{ellipsis_v}**). +1. In the upper-right corner, select the vertical ellipsis (**{ellipsis_v}**). 1. Select **Turn on confidentiality** or **Turn off confidentiality**. ### Who can see confidential OKRs diff --git a/doc/user/project/issues/managing_issues.md b/doc/user/project/issues/managing_issues.md index 6f2d0083ae8..832c1ad63f6 100644 --- a/doc/user/project/issues/managing_issues.md +++ b/doc/user/project/issues/managing_issues.md @@ -355,7 +355,7 @@ To delete an issue: 1. On the left sidebar, select **Search or go to** and find your project. 1. Select **Plan > Issues**, then select your issue to view it. -1. On the top right corner, select **Issue actions** (**{ellipsis_v}**). +1. On the upper-right corner, select **Issue actions** (**{ellipsis_v}**). 1. Select **Delete issue**. Alternatively: @@ -377,7 +377,7 @@ To promote an issue to an epic: 1. On the left sidebar, select **Search or go to** and find your project. 1. Select **Plan > Issues**, then select your issue to view it. -1. On the top right corner, select **Issue actions** (**{ellipsis_v}**). +1. On the upper-right corner, select **Issue actions** (**{ellipsis_v}**). 1. Select **Promote to epic**. Alternatively, you can use the `/promote` [quick action](../quick_actions.md#issues-merge-requests-and-epics). diff --git a/doc/user/project/milestones/index.md b/doc/user/project/milestones/index.md index a959507b338..f4df178794d 100644 --- a/doc/user/project/milestones/index.md +++ b/doc/user/project/milestones/index.md @@ -142,7 +142,7 @@ To edit a milestone: 1. On the left sidebar, select **Search or go to** and find your project or group. 1. Select **Plan > Milestones**. 1. Select a milestone's title. -1. In the top right corner, select **Milestone actions** (**{ellipsis_v}**) and then select **Edit**. +1. In the upper-right corner, select **Milestone actions** (**{ellipsis_v}**) and then select **Edit**. 1. Edit the title, start date, due date, or description. 1. Select **Save changes**. @@ -159,7 +159,7 @@ To edit a milestone: 1. On the left sidebar, select **Search or go to** and find your project or group. 1. Select **Plan > Milestones**. 1. Select a milestone's title. -1. In the top right corner, select **Milestone actions** (**{ellipsis_v}**) and then select **Delete**. +1. In the upper-right corner, select **Milestone actions** (**{ellipsis_v}**) and then select **Delete**. 1. Select **Delete milestone**. ## Promote a project milestone to a group milestone diff --git a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md index 345a30da198..b34369cb3b7 100644 --- a/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md +++ b/doc/user/project/pages/custom_domains_ssl_tls_certification/index.md @@ -47,7 +47,7 @@ this document for an [overview on DNS records](dns_concepts.md). To add your custom domain to GitLab Pages: 1. On the left sidebar, select **Search or go to** and find your project. -1. On the left sidebar, select **Deploy > Pages**. +1. Select **Deploy > Pages**. 1. In the upper-right corner, select **New Domain**. 1. In **Domain**, enter the domain name. 1. Optional. In **Certificate**, turn off the **Automatic certificate management using Let's Encrypt** toggle to add an [SSL/TLS certificate](#adding-an-ssltls-certificate-to-pages). You can also add the certificate and key later. @@ -158,7 +158,7 @@ If you're using Cloudflare, check After you have added all the DNS records: 1. On the left sidebar, select **Search or go to** and find your project. -1. On the left sidebar, select **Deploy > Pages**. +1. Select **Deploy > Pages**. 1. Next to the domain name, select **Edit**. 1. In **Verification status**, select **Retry verification** (**{retry}**). @@ -287,7 +287,7 @@ domain (as long as you've set a valid certificate for it). To enable this setting: 1. On the left sidebar, select **Search or go to** and find your project. -1. On the left sidebar, select **Deploy > Pages**. +1. Select **Deploy > Pages**. 1. Select the **Force HTTPS (requires valid certificates)** checkbox. 1. Select **Save changes**. diff --git a/doc/user/project/repository/branches/index.md b/doc/user/project/repository/branches/index.md index 0a0cbcbb9c8..72d482edb1a 100644 --- a/doc/user/project/repository/branches/index.md +++ b/doc/user/project/repository/branches/index.md @@ -35,7 +35,7 @@ To create a new branch from the GitLab UI: 1. On the left sidebar, select **Search or go to** and find your project. 1. Select **Code > Branches**. -1. On the top right, select **New branch**. +1. On the upper-right corner, select **New branch**. 1. Enter a **Branch name**. 1. In **Create from**, select the base of your branch: an existing branch, an existing tag, or a commit SHA. diff --git a/doc/user/tasks.md b/doc/user/tasks.md index 1ec211dcab3..8fefdf16432 100644 --- a/doc/user/tasks.md +++ b/doc/user/tasks.md @@ -366,7 +366,7 @@ To copy the task reference to your clipboard: 1. On the left sidebar, select **Search or go to** and find your project. 1. Select **Plan > Issues**, then select your issue to view it. 1. In the issue description, in the **Tasks** section, select your task. -1. In the top right corner, select the vertical ellipsis (**{ellipsis_v}**), then select **Copy Reference**. +1. In the upper-right corner, select the vertical ellipsis (**{ellipsis_v}**), then select **Copy Reference**. You can now paste the reference into another description or comment. @@ -386,7 +386,7 @@ To copy the task's email address: 1. On the left sidebar, select **Search or go to** and find your project. 1. Select **Plan > Issues**, then select your issue to view it. -1. In the top right corner, select the vertical ellipsis (**{ellipsis_v}**), then select **Copy task email address**. +1. In the upper-right corner, select the vertical ellipsis (**{ellipsis_v}**), then select **Copy task email address**. ## Set an issue as a parent @@ -442,7 +442,7 @@ Check that box and select **Create task**. To change the confidentiality of an existing task: 1. [Open the task](#view-tasks). -1. In the top right corner, select the vertical ellipsis (**{ellipsis_v}**). +1. In the upper-right corner, select the vertical ellipsis (**{ellipsis_v}**). 1. Select **Turn on confidentiality**. ### Who can see confidential tasks diff --git a/qa/qa/specs/features/sanity/feature_flags_spec.rb b/qa/qa/specs/features/sanity/feature_flags_spec.rb deleted file mode 100644 index 36069558701..00000000000 --- a/qa/qa/specs/features/sanity/feature_flags_spec.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.describe 'Framework sanity', :sanity_feature_flags do - describe 'Feature flag handler checks' do - context 'with an existing feature flag definition file' do - let(:definition) do - path = Pathname.new('../config/feature_flags') - .expand_path(Runtime::Path.qa_root) - .glob('**/*.yml') - .first - YAML.safe_load(File.read(path)) - end - - it 'reads the correct default enabled state' do - # This test will fail if we ever remove all the feature flags, but that's very unlikely given how many there - # are and how much we rely on them. - expect(QA::Runtime::Feature.enabled?(definition['name'])).to be definition['default_enabled'] - end - end - - describe 'feature flag definition files' do - let(:file) do - path = Pathname.new("#{root}/config/feature_flags/development").expand_path(Runtime::Path.qa_root) - path.mkpath - Tempfile.new(%w[ff-test .yml], path) - end - - let(:flag) { Pathname.new(file.path).basename('.yml').to_s } - let(:root) { '..' } - - before do - definition = <<~YAML - name: #{flag} - type: development - default_enabled: #{flag_enabled} - YAML - File.write(file, definition) - end - - after do - file.close! - end - - shared_examples 'gets flag value' do - context 'with a default disabled feature flag' do - let(:flag_enabled) { 'false' } - - it 'reads the flag as disabled' do - expect(QA::Runtime::Feature.enabled?(flag)).to be false - end - - it 'reads as enabled after the flag is enabled' do - QA::Runtime::Feature.enable(flag) - - expect { QA::Runtime::Feature.enabled?(flag) }.to eventually_be_truthy - .within(max_duration: 60, sleep_interval: 5) - end - end - - context 'with a default enabled feature flag' do - let(:flag_enabled) { 'true' } - - it 'reads the flag as enabled' do - expect(QA::Runtime::Feature.enabled?(flag)).to be true - end - - it 'reads as disabled after the flag is disabled' do - QA::Runtime::Feature.disable(flag) - - expect { QA::Runtime::Feature.enabled?(flag) }.to eventually_be_falsey - .within(max_duration: 60, sleep_interval: 5) - end - end - end - - context 'with a CE feature flag' do - include_examples 'gets flag value' - end - - context 'with an EE feature flag' do - let(:root) { '../ee' } - - include_examples 'gets flag value' - end - end - end - end -end diff --git a/qa/qa/specs/features/sanity/framework_spec.rb b/qa/qa/specs/features/sanity/framework_spec.rb deleted file mode 100644 index 5c80afe338e..00000000000 --- a/qa/qa/specs/features/sanity/framework_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.describe 'Framework sanity', :orchestrated, :framework do - describe 'Passing orchestrated example' do - it 'succeeds' do - Runtime::Browser.visit(:gitlab, Page::Main::Login) - - expect(page).to have_text('GitLab') - end - end - - describe 'Failing orchestrated example' do - it 'fails' do - Runtime::Browser.visit(:gitlab, Page::Main::Login) - - expect(page).to have_text("These Aren't the Texts You're Looking For", wait: 1) - end - end - end -end diff --git a/qa/qa/specs/features/sanity/interception_spec.rb b/qa/qa/specs/features/sanity/interception_spec.rb deleted file mode 100644 index 67be832055d..00000000000 --- a/qa/qa/specs/features/sanity/interception_spec.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module QA - RSpec.describe 'Framework sanity', :orchestrated, :framework do - describe 'Browser request interception' do - before(:context) do - skip 'Only can test for chrome' unless QA::Runtime::Env.can_intercept? - end - - before do - Runtime::Browser.visit(:gitlab, Page::Main::Login) - end - - let(:page) { Capybara.current_session } - let(:logger) { class_double('QA::Runtime::Logger') } - - it 'intercepts failed graphql calls' do - page.execute_script <<~JS - fetch('/api/graphql', { - method: 'POST', - body: JSON.stringify({ query: 'query {}'}), - headers: { 'Content-Type': 'application/json' } - }) - JS - - Support::Waiter.wait_until do - !get_cached_error.nil? - end - expect(**get_cached_error).to include({ 'method' => 'POST', 'status' => 200, 'url' => '/api/graphql' }) - end - - def get_cached_error - cache = page.execute_script <<~JS - return Interceptor.getCache() - JS - - cache['errors']&.first - end - end - end -end diff --git a/qa/qa/specs/features/sanity/version_spec.rb b/qa/qa/specs/features/sanity/version_spec.rb deleted file mode 100644 index deefe830c36..00000000000 --- a/qa/qa/specs/features/sanity/version_spec.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -require 'airborne' - -module QA - # This test ensures that the version described by the `DEPLOY_VERSION` - # environment variable is the version actually running. - # - # See https://gitlab.com/gitlab-com/gl-infra/delivery/-/issues/1179 - RSpec.describe 'Framework sanity', :smoke, only: { pipeline: [:pre, :release] } do - describe 'Version check' do - let(:api_client) { Runtime::API::Client.new(:gitlab) } - let(:request) { Runtime::API::Request.new(api_client, '/version') } - - it 'is the specified version' do - # The `DEPLOY_VERSION` variable will only be provided for deploys to the - # `pre` and `release` environments, which only receive packaged releases. - # - # For these releases, `deploy_version` will be a package string (e.g., - # `13.1.3-ee.0`), and the reported version will be something like - # `13.1.3-ee`, so we only compare the leading SemVer string. - # - # | Package | Version | - # | ---------------- | -------------- | - # | 13.3.5-ee.0 | 13.3.5-ee | - # | 13.3.0-rc42.ee.0 | 13.3.0-rc42-ee | - deploy = Runtime::Env.deploy_version&.gsub(/\A(\d+\.\d+\.\d+).*\z/, '\1') - - skip('No deploy version provided') if deploy.nil? || deploy.empty? - - get request.url - - expect_status(200) - expect(json_body).to have_key(:version) - expect(json_body[:version]).to start_with(deploy) - end - end - end -end diff --git a/qa/qa/tools/reliable_report.rb b/qa/qa/tools/reliable_report.rb index 7569095b56f..4f9d166cac6 100644 --- a/qa/qa/tools/reliable_report.rb +++ b/qa/qa/tools/reliable_report.rb @@ -29,7 +29,8 @@ module QA # Project for report creation: https://gitlab.com/gitlab-org/gitlab PROJECT_ID = 278964 - FEATURES_DIR = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/' + BLOB_MASTER = 'https://gitlab.com/gitlab-org/gitlab/-/blob/master' + FEATURES_DIR = '/qa/qa/specs/features' # @param [Integer] range amount of days for results range def initialize(range) @@ -49,6 +50,7 @@ module QA if report_in_issue_and_slack == "true" reporter.report_in_issue_and_slack + reporter.write_specs_json reporter.close_previous_reports end rescue StandardError => e @@ -83,8 +85,8 @@ module QA labels: "#{RELIABLE_REPORT_LABEL},Quality,test,type::maintenance,automation:ml" ) @report_iid = issue[:iid] - web_url = issue[:web_url] - puts "Created report issue: #{web_url}" + @report_web_url = issue[:web_url] + puts "Created report issue: #{@report_web_url}" puts "Sending slack notification".colorize(:green) notifier.post( @@ -93,7 +95,7 @@ module QA ```#{summary_table(stable: true)}``` ```#{summary_table(stable: false)}``` - #{web_url} + #{@report_web_url} TEXT ) puts "Done!" @@ -129,9 +131,14 @@ module QA ) end + def write_specs_json + File.write('tmp/unstable_specs.json', JSON.pretty_generate(specs_attributes(stable: false))) + File.write('tmp/stable_specs.json', JSON.pretty_generate(specs_attributes(stable: true))) + end + private - attr_reader :range, :slack_channel, :report_iid + attr_reader :range, :slack_channel, :report_iid, :report_web_url # Slack notifier # @@ -371,24 +378,103 @@ module QA end.join('')}" end + def api_query_unreliable + @api_query_unreliable ||= begin + log_fetching_query_data(true) + query_api.query(query: query(false)) + end + end + + def api_query_reliable + @api_query_reliable ||= begin + log_fetching_query_data(true) + query_api.query(query: query(true)) + end + end + + def log_fetching_query_data(reliable) + puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past #{range} days\n".colorize(:green)) + end + + def specs_attributes(stable:) + all_runs = stable ? api_query_unreliable : api_query_reliable + + specs_array = all_runs.each_with_object([]) do |table, arr| + records = table.records.sort_by { |record| record.values["_time"] } + + next if within_execution_range(records.first.values["_time"], records.last.values["_time"]) + + result = spec_attributes_per_run(records) + + next if !stable && result[:failed] == 0 + + next if stable && result[:failed] != 0 + + # A failure issue does not exist + next if !stable && result[:failure_issue].exclude?('issues') + + arr << result + end + + { + type: stable ? 'Stable Specs' : 'Unstable Specs', + report_issue: report_web_url, + specs: specs_array + } + end + + def spec_attributes_per_run(records) + failed = records.count do |r| + r.values["status"] == "failed" && !allowed_failure?(r.values["failure_exception"]) + end + + failure_issue = exceptions_and_related_urls(records).values.last + last_record = records.last.values + name = last_record["name"] + file = last_record["file_path"].split("/").last + link = BLOB_MASTER + FEATURES_DIR + last_record["file_path"] + file_path = FEATURES_DIR + last_record["file_path"] + stage = last_record["stage"] || "unknown" + testcase = last_record["testcase"] + run_type = last_record["run_type"] + product_group = last_record["product_group"] || "unknown" + runs = records.count + failure_rate = (failed.to_f / runs) * 100 + + { + stage: stage, + product_group: product_group, + name: name, + file: file, + link: link, + runs: runs, + failed: failed, + failure_issue: failure_issue || '', + failure_rate: failure_rate == 0 ? failure_rate.round(0) : failure_rate.round(2), + testcase: testcase, + file_path: file_path, + run_type: run_type + } + end + # rubocop:disable Metrics/AbcSize # Test executions grouped by name # # @param [Boolean] reliable # @return [Hash] def test_runs(reliable:) - puts("Fetching data on #{reliable ? 'reliable ' : ''}test execution for past #{range} days\n".colorize(:green)) + all_runs = reliable ? api_query_reliable : api_query_unreliable - all_runs = query_api.query(query: query(reliable)) all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result| records = table.records.sort_by { |record| record.values["_time"] } next if within_execution_range(records.first.values["_time"], records.last.values["_time"]) last_record = records.last.values + name = last_record["name"] file = last_record["file_path"].split("/").last - link = FEATURES_DIR + last_record["file_path"] + link = BLOB_MASTER + FEATURES_DIR + last_record["file_path"] stage = last_record["stage"] || "unknown" product_group = last_record["product_group"] || "unknown" diff --git a/qa/spec/tools/reliable_report_spec.rb b/qa/spec/tools/reliable_report_spec.rb index c8488424259..9fc94894bcd 100644 --- a/qa/spec/tools/reliable_report_spec.rb +++ b/qa/spec/tools/reliable_report_spec.rb @@ -3,177 +3,199 @@ describe QA::Tools::ReliableReport do include QA::Support::Helpers::StubEnv - describe '.run' do - subject(:run) { described_class.run(range: range, report_in_issue_and_slack: create_issue) } + before do + stub_env("QA_INFLUXDB_URL", "url") + stub_env("QA_INFLUXDB_TOKEN", "token") + stub_env("SLACK_WEBHOOK", "slack_url") + stub_env("CI_API_V4_URL", "gitlab_api_url") + stub_env("GITLAB_ACCESS_TOKEN", "gitlab_token") + + allow(RestClient::Request).to receive(:execute) + allow(Slack::Notifier).to receive(:new).and_return(slack_notifier) + allow(InfluxDB2::Client).to receive(:new).and_return(influx_client) + + allow(query_api).to receive(:query).with(query: flux_query(reliable: false)).and_return(runs) + allow(query_api).to receive(:query).with(query: flux_query(reliable: true)).and_return(reliable_runs) + end - let(:slack_notifier) { instance_double("Slack::Notifier", post: nil) } - let(:influx_client) { instance_double("InfluxDB2::Client", create_query_api: query_api) } - let(:query_api) { instance_double("InfluxDB2::QueryApi") } - - let(:slack_channel) { "#quality-reports" } - let(:range) { 14 } - let(:issue_url) { "https://gitlab.com/issue/1" } - let(:time) { "2021-12-07T04:05:25.000000000+00:00" } - let(:failure_message) { 'random failure message' } - - let(:runs) do - values = { - "name" => "stable spec1", - "status" => "passed", - "file_path" => "some/spec.rb", - "stage" => "create", - "product_group" => "code_review", - "_time" => time - } - more_values = { - "name" => "stable spec2", - "status" => "passed", - "file_path" => "some/spec.rb", - "stage" => "manage", - "product_group" => "import_and_integrate", - "_time" => time - } - [ - instance_double( - "InfluxDB2::FluxTable", - records: [ - instance_double("InfluxDB2::FluxRecord", values: values), - instance_double("InfluxDB2::FluxRecord", values: values), - instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s })) - ] - ), - instance_double( - "InfluxDB2::FluxTable", - records: [ - instance_double("InfluxDB2::FluxRecord", values: more_values), - instance_double("InfluxDB2::FluxRecord", values: more_values), - instance_double("InfluxDB2::FluxRecord", values: more_values.merge({ "_time" => Time.now.to_s })) - ] - ) - ] - end + let(:slack_notifier) { instance_double("Slack::Notifier", post: nil) } + let(:influx_client) { instance_double("InfluxDB2::Client", create_query_api: query_api) } + let(:query_api) { instance_double("InfluxDB2::QueryApi") } + + let(:slack_channel) { "#quality-reports" } + let(:range) { 14 } + let(:issue_url) { "https://gitlab.com/issue/1" } + let(:time) { "2021-12-07T04:05:25.000000000+00:00" } + let(:failure_message) { 'random failure message' } + + let(:run_values) do + { + "name" => "stable spec1", + "status" => "passed", + "file_path" => "/some/spec.rb", + "stage" => "create", + "product_group" => "code_review", + "testcase" => "https://testcase/url", + "run_type" => "staging", + "_time" => time + } + end - let(:reliable_runs) do - values = { - "name" => "unstable spec", - "status" => "failed", - "file_path" => "some/spec.rb", - "stage" => "create", - "product_group" => "code_review", - "failure_exception" => failure_message, - "job_url" => "https://job/url", - "_time" => time - } - more_values = { - "name" => "unstable spec", - "status" => "failed", - "file_path" => "some/spec.rb", - "stage" => "manage", - "product_group" => "import_and_integrate", - "failure_exception" => failure_message, - "job_url" => "https://job/url", - "_time" => time - } - [ - instance_double( - "InfluxDB2::FluxTable", - records: [ - instance_double("InfluxDB2::FluxRecord", values: { **values, "status" => "passed" }), - instance_double("InfluxDB2::FluxRecord", values: values), - instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s })) - ] - ), - instance_double( - "InfluxDB2::FluxTable", - records: [ - instance_double("InfluxDB2::FluxRecord", values: { **more_values, "status" => "passed" }), - instance_double("InfluxDB2::FluxRecord", values: more_values), - instance_double("InfluxDB2::FluxRecord", values: more_values.merge({ "_time" => Time.now.to_s })) - ] - ) - ] - end + let(:run_more_values) do + { + "name" => "stable spec2", + "status" => "passed", + "file_path" => "/some/spec.rb", + "stage" => "manage", + "product_group" => "import_and_integrate", + "testcase" => "https://testcase/url", + "run_type" => "staging", + "_time" => time + } + end - def flux_query(reliable:) - <<~QUERY - from(bucket: "e2e-test-stats-main") - |> range(start: -#{range}d) - |> filter(fn: (r) => r._measurement == "test-stats") - |> filter(fn: (r) => r.run_type == "staging-full" or - r.run_type == "staging-sanity" or - r.run_type == "production-full" or - r.run_type == "production-sanity" or - r.run_type == "package-and-qa" or - r.run_type == "nightly" - ) - |> filter(fn: (r) => r.job_name != "airgapped" and - r.job_name != "instance-image-slow-network" and - r.job_name != "nplus1-instance-image" - ) - |> filter(fn: (r) => r.status != "pending" and - r.merge_request == "false" and - r.quarantined == "false" and - r.smoke == "false" and - r.reliable == "#{reliable}" - ) - |> filter(fn: (r) => r["_field"] == "job_url" or - r["_field"] == "failure_exception" or - r["_field"] == "id" or - r["_field"] == "failure_issue" - ) - |> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value") - |> group(columns: ["name"]) - QUERY - end + let(:runs) do + [ + instance_double( + "InfluxDB2::FluxTable", + records: [ + instance_double("InfluxDB2::FluxRecord", values: run_values), + instance_double("InfluxDB2::FluxRecord", values: run_values), + instance_double("InfluxDB2::FluxRecord", values: run_values.merge({ "_time" => Time.now.to_s })) + ] + ), + instance_double( + "InfluxDB2::FluxTable", + records: [ + instance_double("InfluxDB2::FluxRecord", values: run_more_values), + instance_double("InfluxDB2::FluxRecord", values: run_more_values), + instance_double("InfluxDB2::FluxRecord", values: run_more_values.merge({ "_time" => Time.now.to_s })) + ] + ) + ] + end - def expected_stage_markdown(result, stage, product_group, type) - <<~SECTION.strip - ## #{stage.capitalize} (1) + let(:reliable_run_values) do + { + "name" => "unstable spec", + "status" => "failed", + "file_path" => "/some/spec.rb", + "stage" => "create", + "product_group" => "code_review", + "failure_exception" => failure_message, + "job_url" => "https://job/url", + "testcase" => "https://testcase/url", + "failure_issue" => "https://issues/url", + "run_type" => "staging", + "_time" => time + } + end -
- Executions table ~\"group::#{product_group}\" (1) + let(:reliable_run_more_values) do + { + "name" => "unstable spec", + "status" => "failed", + "file_path" => "/some/spec.rb", + "stage" => "manage", + "product_group" => "import_and_integrate", + "failure_exception" => failure_message, + "job_url" => "https://job/url", + "testcase" => "https://testcase/url", + "failure_issue" => "https://issues/url", + "run_type" => "staging", + "_time" => time + } + end - #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days", true)} + let(:reliable_runs) do + [ + instance_double( + "InfluxDB2::FluxTable", + records: [ + instance_double("InfluxDB2::FluxRecord", values: { **reliable_run_values, "status" => "passed" }), + instance_double("InfluxDB2::FluxRecord", values: reliable_run_values), + instance_double("InfluxDB2::FluxRecord", values: reliable_run_values.merge({ "_time" => Time.now.to_s })) + ] + ), + instance_double( + "InfluxDB2::FluxTable", + records: [ + instance_double("InfluxDB2::FluxRecord", values: { **reliable_run_more_values, "status" => "passed" }), + instance_double("InfluxDB2::FluxRecord", values: reliable_run_more_values), + instance_double("InfluxDB2::FluxRecord", values: reliable_run_more_values.merge({ "_time" => Time.now.to_s })) + ] + ) + ] + end -
- SECTION - end + def flux_query(reliable:) + <<~QUERY + from(bucket: "e2e-test-stats-main") + |> range(start: -#{range}d) + |> filter(fn: (r) => r._measurement == "test-stats") + |> filter(fn: (r) => r.run_type == "staging-full" or + r.run_type == "staging-sanity" or + r.run_type == "production-full" or + r.run_type == "production-sanity" or + r.run_type == "package-and-qa" or + r.run_type == "nightly" + ) + |> filter(fn: (r) => r.job_name != "airgapped" and + r.job_name != "instance-image-slow-network" and + r.job_name != "nplus1-instance-image" + ) + |> filter(fn: (r) => r.status != "pending" and + r.merge_request == "false" and + r.quarantined == "false" and + r.smoke == "false" and + r.reliable == "#{reliable}" + ) + |> filter(fn: (r) => r["_field"] == "job_url" or + r["_field"] == "failure_exception" or + r["_field"] == "id" or + r["_field"] == "failure_issue" + ) + |> pivot(rowKey: ["_time"], columnKey: ["_field"], valueColumn: "_value") + |> group(columns: ["name"]) + QUERY + end - def expected_summary_table(summary, type, markdown = false) - table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50), markdown) - end + def expected_stage_markdown(result, stage, product_group, type) + <<~SECTION.strip + ## #{stage.capitalize} (1) - def table(rows, headings, title, markdown = false) - Terminal::Table.new( - headings: headings, - title: markdown ? nil : title, - rows: rows, - style: markdown ? { border: :markdown } : { all_separators: true } - ) - end +
+ Executions table ~\"group::#{product_group}\" (1) - def name_column(spec_name, exceptions_and_related_urls = {}) - "**Name**: #{spec_name}
**File**: [spec.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/some/spec.rb)#{exceptions_markdown(exceptions_and_related_urls)}" - end + #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days", true)} - def exceptions_markdown(exceptions_and_related_urls) - exceptions_and_related_urls.empty? ? '' : "
**Exceptions**:
- [`#{failure_message}`](https://job/url)" - end +
+ SECTION + end - before do - stub_env("QA_INFLUXDB_URL", "url") - stub_env("QA_INFLUXDB_TOKEN", "token") - stub_env("SLACK_WEBHOOK", "slack_url") - stub_env("CI_API_V4_URL", "gitlab_api_url") - stub_env("GITLAB_ACCESS_TOKEN", "gitlab_token") - - allow(RestClient::Request).to receive(:execute) - allow(Slack::Notifier).to receive(:new).and_return(slack_notifier) - allow(InfluxDB2::Client).to receive(:new).and_return(influx_client) - - allow(query_api).to receive(:query).with(query: flux_query(reliable: false)).and_return(runs) - allow(query_api).to receive(:query).with(query: flux_query(reliable: true)).and_return(reliable_runs) - end + def expected_summary_table(summary, type, markdown = false) + table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50), markdown) + end + + def table(rows, headings, title, markdown = false) + Terminal::Table.new( + headings: headings, + title: markdown ? nil : title, + rows: rows, + style: markdown ? { border: :markdown } : { all_separators: true } + ) + end + + def name_column(spec_name, exceptions_and_related_urls = {}) + "**Name**: #{spec_name}
**File**: [spec.rb](https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/some/spec.rb)#{exceptions_markdown(exceptions_and_related_urls)}" + end + + def exceptions_markdown(exceptions_and_related_urls) + exceptions_and_related_urls.empty? ? '' : "
**Exceptions**:
- [`#{failure_message}`](https://issues/url)" + end + + describe '.run' do + subject(:run) { described_class.run(range: range, report_in_issue_and_slack: create_issue) } context "without report creation" do let(:create_issue) { "false" } @@ -398,35 +420,125 @@ describe QA::Tools::ReliableReport do it "returns an empty hash" do expect(reliable_report.send(:exceptions_and_related_urls, records)).to be_empty end + end - context "with failure_exception" do - context "without failure_issue" do - let(:values) do - { - "failure_exception" => failure_message, - "job_url" => job_url - } - end - - it "returns job_url as value" do - expect(reliable_report.send(:exceptions_and_related_urls, records).values).to eq([job_url]) - end + context "with failure_exception" do + context "without failure_issue" do + let(:values) do + { + "failure_exception" => failure_message, + "job_url" => job_url + } end - context "with failure_issue and job_url" do - let(:values) do - { - "failure_exception" => failure_message, - "failure_issue" => failure_issue_url, - "job_url" => job_url - } - end - - it "returns failure_issue as value" do - expect(reliable_report.send(:exceptions_and_related_urls, records).values).to eq([failure_issue_url]) - end + it "returns job_url as value" do + expect(reliable_report.send(:exceptions_and_related_urls, records).values).to eq([job_url]) end end + + context "with failure_issue and job_url" do + let(:values) do + { + "failure_exception" => failure_message, + "failure_issue" => failure_issue_url, + "job_url" => job_url + } + end + + it "returns failure_issue as value" do + expect(reliable_report.send(:exceptions_and_related_urls, records).values).to eq([failure_issue_url]) + end + end + end + end + + describe "#specs_attributes" do + subject(:reliable_report) { described_class.new(14) } + + let(:report_web_url) { 'https://report/url' } + + before do + allow(reliable_report).to receive(:report_web_url).and_return(report_web_url) + end + + shared_examples "spec attributes" do |stable| + it "returns #{stable} spec attributes" do + expect(reliable_report.send(:specs_attributes, stable: stable)).to eq(expected_specs_attributes) + end + end + + context "with stable false" do + let(:expected_specs_attributes) do + { type: "Unstable Specs", + report_issue: "https://report/url", + specs: + [ + { stage: "create", + product_group: "code_review", + name: "unstable spec", + file: "spec.rb", + link: "https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/some/spec.rb", + runs: 3, + failed: 2, + failure_issue: "https://issues/url", + failure_rate: 66.67, + testcase: "https://testcase/url", + file_path: "/qa/qa/specs/features/some/spec.rb", + run_type: "staging" }, + { stage: "manage", + product_group: "import_and_integrate", + name: "unstable spec", + file: "spec.rb", + link: "https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/some/spec.rb", + runs: 3, + failed: 2, + failure_issue: "https://issues/url", + failure_rate: 66.67, + testcase: "https://testcase/url", + file_path: "/qa/qa/specs/features/some/spec.rb", + run_type: "staging" } + ] } + end + + it_behaves_like "spec attributes", false + end + + context "with stable true" do + let(:expected_specs_attributes) do + { + type: "Stable Specs", + report_issue: "https://report/url", + specs: + [ + { stage: "create", + product_group: "code_review", + name: "stable spec1", + file: "spec.rb", + link: "https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/some/spec.rb", + runs: 3, + failed: 0, + failure_issue: "", + failure_rate: 0, + testcase: "https://testcase/url", + file_path: "/qa/qa/specs/features/some/spec.rb", + run_type: "staging" }, + { stage: "manage", + product_group: "import_and_integrate", + name: "stable spec2", + file: "spec.rb", + link: "https://gitlab.com/gitlab-org/gitlab/-/blob/master/qa/qa/specs/features/some/spec.rb", + runs: 3, + failed: 0, + failure_issue: "", + failure_rate: 0, + testcase: "https://testcase/url", + file_path: "/qa/qa/specs/features/some/spec.rb", + run_type: "staging" } + ] + } + end + + it_behaves_like "spec attributes", true end end end -- cgit v1.2.3