diff options
34 files changed, 218 insertions, 100 deletions
diff --git a/.gitlab/ci/package-and-test/main.gitlab-ci.yml b/.gitlab/ci/package-and-test/main.gitlab-ci.yml index 79f31a46d3e..21dd8f957d4 100644 --- a/.gitlab/ci/package-and-test/main.gitlab-ci.yml +++ b/.gitlab/ci/package-and-test/main.gitlab-ci.yml @@ -407,9 +407,7 @@ integrations: - !reference [.rules:test:manual, rules] ldap-no-server: - extends: - - .qa - - .failure-videos + extends: .qa variables: QA_SCENARIO: Test::Integration::LDAPNoServer rules: @@ -418,9 +416,7 @@ ldap-no-server: - !reference [.rules:test:manual, rules] ldap-tls: - extends: - - .qa - - .failure-videos + extends: .qa variables: QA_SCENARIO: Test::Integration::LDAPTLS rules: @@ -429,9 +425,7 @@ ldap-tls: - !reference [.rules:test:manual, rules] ldap-no-tls: - extends: - - .qa - - .failure-videos + extends: .qa variables: QA_SCENARIO: Test::Integration::LDAPNoTLS rules: diff --git a/.rubocop_todo/layout/line_length.yml b/.rubocop_todo/layout/line_length.yml index 4251f764e78..8365d314f3a 100644 --- a/.rubocop_todo/layout/line_length.yml +++ b/.rubocop_todo/layout/line_length.yml @@ -4921,7 +4921,6 @@ Layout/LineLength: - 'spec/workers/auto_devops/disable_worker_spec.rb' - 'spec/workers/bulk_import_worker_spec.rb' - 'spec/workers/bulk_imports/export_request_worker_spec.rb' - - 'spec/workers/bulk_imports/stuck_import_worker_spec.rb' - 'spec/workers/ci/job_artifacts/expire_project_build_artifacts_worker_spec.rb' - 'spec/workers/ci/merge_requests/add_todo_when_build_fails_worker_spec.rb' - 'spec/workers/ci/pending_builds/update_group_worker_spec.rb' diff --git a/app/graphql/resolvers/packages_base_resolver.rb b/app/graphql/resolvers/packages_base_resolver.rb index 084fbc23156..7e5d89a7897 100644 --- a/app/graphql/resolvers/packages_base_resolver.rb +++ b/app/graphql/resolvers/packages_base_resolver.rb @@ -20,7 +20,8 @@ module Resolvers default_value: nil argument :package_version, GraphQL::Types::String, - description: 'Filter a package by version.', + description: 'Filter a package by version. If used in combination with `include_versionless`, + then no versionless packages are returned.', required: false, default_value: nil diff --git a/app/services/bulk_imports/file_download_service.rb b/app/services/bulk_imports/file_download_service.rb index 1f2437d783d..cc2d544198b 100644 --- a/app/services/bulk_imports/file_download_service.rb +++ b/app/services/bulk_imports/file_download_service.rb @@ -83,7 +83,7 @@ module BulkImports end def raise_error(message) - logger.warn(message: message, response_headers: response_headers, importer: 'gitlab_migration') + logger.warn(message: message, response_headers: response_headers) raise ServiceError, message end @@ -112,7 +112,7 @@ module BulkImports end def logger - @logger ||= Gitlab::Import::Logger.build + @logger ||= Logger.build end def validate_url diff --git a/app/services/bulk_imports/process_service.rb b/app/services/bulk_imports/process_service.rb index ebf075f0669..7a6a883f1a9 100644 --- a/app/services/bulk_imports/process_service.rb +++ b/app/services/bulk_imports/process_service.rb @@ -113,13 +113,12 @@ module BulkImports pipeline_class: pipeline[:pipeline], minimum_source_version: minimum_version, maximum_source_version: maximum_version, - source_version: entity.source_version.to_s, - importer: 'gitlab_migration' + source_version: entity.source_version.to_s ) end def logger - @logger ||= Gitlab::Import::Logger.build + @logger ||= Logger.build end end end diff --git a/app/views/events/event/_private.html.haml b/app/views/events/event/_private.html.haml index d91f30c07cb..5e9d6da3996 100644 --- a/app/views/events/event/_private.html.haml +++ b/app/views/events/event/_private.html.haml @@ -1,8 +1,8 @@ -.event-item - .event-item-timestamp +.event-item{ class: current_path?('users#activity') ? 'user-profile-activity gl-border-bottom-0 gl-pl-7! gl-pb-3' : '' } + .event-item-timestamp.gl-font-sm = time_ago_with_tooltip(event.created_at) - .system-note-image= sprite_icon('eye-slash', size: 24, css_class: 'icon') + .system-note-image.gl-rounded-full.gl-bg-gray-50.gl-line-height-0= sprite_icon('eye-slash', size: 14, css_class: 'icon') = event_user_info(event) diff --git a/app/workers/bulk_imports/entity_worker.rb b/app/workers/bulk_imports/entity_worker.rb index b35fa2ced3f..e510a8c0d06 100644 --- a/app/workers/bulk_imports/entity_worker.rb +++ b/app/workers/bulk_imports/entity_worker.rb @@ -79,7 +79,7 @@ module BulkImports end def logger - @logger ||= Gitlab::Import::Logger.build + @logger ||= Logger.build end def log_exception(exception, payload) @@ -99,7 +99,7 @@ module BulkImports bulk_import_entity_type: entity.source_type, source_full_path: entity.source_full_path, source_version: source_version, - importer: 'gitlab_migration' + importer: Logger::IMPORTER_NAME } defaults.merge(extra) diff --git a/app/workers/bulk_imports/export_request_worker.rb b/app/workers/bulk_imports/export_request_worker.rb index 44759916f99..f7456ddccb1 100644 --- a/app/workers/bulk_imports/export_request_worker.rb +++ b/app/workers/bulk_imports/export_request_worker.rb @@ -80,8 +80,7 @@ module BulkImports bulk_import_id: entity.bulk_import_id, bulk_import_entity_type: entity.source_type, source_full_path: entity.source_full_path, - source_version: entity.bulk_import.source_version_info.to_s, - importer: 'gitlab_migration' + source_version: entity.bulk_import.source_version_info.to_s } ) @@ -97,7 +96,7 @@ module BulkImports end def logger - @logger ||= Gitlab::Import::Logger.build + @logger ||= Logger.build end def log_exception(exception, payload) @@ -114,8 +113,7 @@ module BulkImports bulk_import_entity_type: entity.source_type, source_full_path: entity.source_full_path, message: "Request to export #{entity.source_type} failed", - source_version: entity.bulk_import.source_version_info.to_s, - importer: 'gitlab_migration' + source_version: entity.bulk_import.source_version_info.to_s } ) diff --git a/app/workers/bulk_imports/finish_batched_pipeline_worker.rb b/app/workers/bulk_imports/finish_batched_pipeline_worker.rb index b953f8ab786..973889e1411 100644 --- a/app/workers/bulk_imports/finish_batched_pipeline_worker.rb +++ b/app/workers/bulk_imports/finish_batched_pipeline_worker.rb @@ -22,9 +22,11 @@ module BulkImports return re_enqueue if import_in_progress? if tracker.stale? + logger.error(log_attributes(message: 'Tracker stale. Failing batches and tracker')) tracker.batches.map(&:fail_op!) tracker.fail_op! else + logger.info(log_attributes(message: 'Tracker finished')) tracker.finish! end end @@ -40,5 +42,20 @@ module BulkImports def import_in_progress? tracker.batches.any? { |b| b.started? || b.created? } end + + def logger + @logger ||= Logger.build + end + + def log_attributes(extra = {}) + structured_payload( + { + tracker_id: tracker.id, + bulk_import_id: tracker.entity.id, + bulk_import_entity_id: tracker.entity.bulk_import_id, + pipeline_class: tracker.pipeline_name + }.merge(extra) + ) + end end end diff --git a/app/workers/bulk_imports/pipeline_batch_worker.rb b/app/workers/bulk_imports/pipeline_batch_worker.rb index f5e2efcff97..55936e85d48 100644 --- a/app/workers/bulk_imports/pipeline_batch_worker.rb +++ b/app/workers/bulk_imports/pipeline_batch_worker.rb @@ -45,9 +45,11 @@ module BulkImports def run return batch.skip! if tracker.failed? || tracker.finished? + logger.info(log_attributes(message: 'Batch tracker started')) batch.start! tracker.pipeline_class.new(context).run batch.finish! + logger.info(log_attributes(message: 'Batch tracker finished')) rescue BulkImports::RetryPipelineError => e @pending_retry = true retry_batch(e) @@ -56,13 +58,7 @@ module BulkImports def fail_batch(exception) batch.fail_op! - Gitlab::ErrorTracking.track_exception( - exception, - batch_id: batch.id, - tracker_id: tracker.id, - pipeline_class: tracker.pipeline_name, - pipeline_step: 'pipeline_batch_worker_run' - ) + Gitlab::ErrorTracking.track_exception(exception, log_attributes(message: 'Batch tracker failed')) BulkImports::Failure.create( bulk_import_entity_id: batch.tracker.entity.id, @@ -103,5 +99,24 @@ module BulkImports def process_batch? batch.created? || batch.started? end + + def logger + @logger ||= Logger.build + end + + def log_attributes(extra = {}) + structured_payload( + { + batch_id: batch.id, + batch_number: batch.batch_number, + tracker_id: tracker.id, + bulk_import_id: tracker.entity.bulk_import_id, + bulk_import_entity_id: tracker.entity.id, + pipeline_class: tracker.pipeline_name, + pipeline_step: 'pipeline_batch_worker_run', + importer: Logger::IMPORTER_NAME + }.merge(extra) + ) + end end end diff --git a/app/workers/bulk_imports/pipeline_worker.rb b/app/workers/bulk_imports/pipeline_worker.rb index 7573ce9c3ac..4b1df9c85a6 100644 --- a/app/workers/bulk_imports/pipeline_worker.rb +++ b/app/workers/bulk_imports/pipeline_worker.rb @@ -97,7 +97,7 @@ module BulkImports end def logger - @logger ||= Gitlab::Import::Logger.build + @logger ||= Logger.build end def re_enqueue(delay = FILE_EXTRACTION_PIPELINE_PERFORM_DELAY) @@ -170,7 +170,7 @@ module BulkImports pipeline_class: pipeline_tracker.pipeline_name, pipeline_tracker_state: pipeline_tracker.human_status_name, source_version: source_version, - importer: 'gitlab_migration' + importer: Logger::IMPORTER_NAME }.merge(extra) ) end diff --git a/app/workers/bulk_imports/stuck_import_worker.rb b/app/workers/bulk_imports/stuck_import_worker.rb index 3fa4221728b..6c8569b0aa0 100644 --- a/app/workers/bulk_imports/stuck_import_worker.rb +++ b/app/workers/bulk_imports/stuck_import_worker.rb @@ -14,18 +14,29 @@ module BulkImports def perform BulkImport.stale.find_each do |import| + logger.error(message: 'BulkImport stale', bulk_import_id: import.id) import.cleanup_stale end - BulkImports::Entity.includes(:trackers).stale.find_each do |import| # rubocop: disable CodeReuse/ActiveRecord + BulkImports::Entity.includes(:trackers).stale.find_each do |entity| # rubocop: disable CodeReuse/ActiveRecord ApplicationRecord.transaction do - import.cleanup_stale + logger.error( + message: 'BulkImports::Entity stale', + bulk_import_id: entity.bulk_import_id, + bulk_import_entity_id: entity.id + ) - import.trackers.find_each do |tracker| + entity.cleanup_stale + + entity.trackers.find_each do |tracker| tracker.cleanup_stale end end end end + + def logger + @logger ||= Logger.build + end end end diff --git a/config/feature_flags/development/restrict_ci_job_token_for_public_and_internal_projects.yml b/config/feature_flags/development/restrict_ci_job_token_for_public_and_internal_projects.yml index a9314190dee..31216be0a5c 100644 --- a/config/feature_flags/development/restrict_ci_job_token_for_public_and_internal_projects.yml +++ b/config/feature_flags/development/restrict_ci_job_token_for_public_and_internal_projects.yml @@ -5,4 +5,4 @@ rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/417172 milestone: '16.6' type: development group: group::pipeline security -default_enabled: false +default_enabled: true diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 965f9f52327..b68d068c3ec 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -18954,7 +18954,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="grouppackagesincludeversionless"></a>`includeVersionless` | [`Boolean`](#boolean) | Include versionless packages. | | <a id="grouppackagespackagename"></a>`packageName` | [`String`](#string) | Search a package by name. | | <a id="grouppackagespackagetype"></a>`packageType` | [`PackageTypeEnum`](#packagetypeenum) | Filter a package by type. | -| <a id="grouppackagespackageversion"></a>`packageVersion` | [`String`](#string) | Filter a package by version. | +| <a id="grouppackagespackageversion"></a>`packageVersion` | [`String`](#string) | Filter a package by version. If used in combination with `include_versionless`, then no versionless packages are returned. | | <a id="grouppackagessort"></a>`sort` | [`PackageGroupSort`](#packagegroupsort) | Sort packages by this criteria. | | <a id="grouppackagesstatus"></a>`status` | [`PackageStatus`](#packagestatus) | Filter a package by status. | @@ -23912,7 +23912,7 @@ four standard [pagination arguments](#connection-pagination-arguments): | <a id="projectpackagesincludeversionless"></a>`includeVersionless` | [`Boolean`](#boolean) | Include versionless packages. | | <a id="projectpackagespackagename"></a>`packageName` | [`String`](#string) | Search a package by name. | | <a id="projectpackagespackagetype"></a>`packageType` | [`PackageTypeEnum`](#packagetypeenum) | Filter a package by type. | -| <a id="projectpackagespackageversion"></a>`packageVersion` | [`String`](#string) | Filter a package by version. | +| <a id="projectpackagespackageversion"></a>`packageVersion` | [`String`](#string) | Filter a package by version. If used in combination with `include_versionless`, then no versionless packages are returned. | | <a id="projectpackagessort"></a>`sort` | [`PackageSort`](#packagesort) | Sort packages by this criteria. | | <a id="projectpackagesstatus"></a>`status` | [`PackageStatus`](#packagestatus) | Filter a package by status. | diff --git a/doc/api/packages.md b/doc/api/packages.md index 21d8d6f5a21..7c8dfeb8710 100644 --- a/doc/api/packages.md +++ b/doc/api/packages.md @@ -30,7 +30,7 @@ GET /projects/:id/packages | `sort` | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. | | `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, `nuget`, `helm`, `terraform_module`, or `golang`. | | `package_name` | string | no | Filter the project packages with a fuzzy search by name. | -| `package_version` | string | no | Filter the project packages by version. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349065) in GitLab 16.6. | +| `package_version` | string | no | Filter the project packages by version. If used in combination with `include_versionless`, then no versionless packages are returned. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349065) in GitLab 16.6. | | `include_versionless` | boolean | no | When set to true, versionless packages are included in the response. | | `status` | string | no | Filter the returned packages by status. One of `default` (default), `hidden`, `processing`, `error`, or `pending_destruction`. | @@ -96,7 +96,7 @@ GET /groups/:id/packages | `sort` | string | no | The direction of the order, either `asc` (default) for ascending order or `desc` for descending order. | | `package_type` | string | no | Filter the returned packages by type. One of `conan`, `maven`, `npm`, `pypi`, `composer`, `nuget`, `helm`, or `golang`. | | `package_name` | string | no | Filter the project packages with a fuzzy search by name. | -| `package_version` | string | no | Filter the returned packages by version. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349065) in GitLab 16.6. | +| `package_version` | string | no | Filter the returned packages by version. If used in combination with `include_versionless`, then no versionless packages are returned. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/349065) in GitLab 16.6. | | `include_versionless` | boolean | no | When set to true, versionless packages are included in the response. | | `status` | string | no | Filter the returned packages by status. One of `default` (default), `hidden`, `processing`, `error`, or `pending_destruction`. | diff --git a/doc/user/application_security/dependency_scanning/index.md b/doc/user/application_security/dependency_scanning/index.md index 787552ef717..8fa17e33df6 100644 --- a/doc/user/application_security/dependency_scanning/index.md +++ b/doc/user/application_security/dependency_scanning/index.md @@ -230,7 +230,8 @@ table.supported-languages ul { <li> <a id="notes-regarding-supported-languages-and-package-managers-2"></a> <p> - Java 21 LTS is only available when using <a href="https://maven.apache.org/">Maven</a> or <a href="https://gradle.org/">Gradle</a>. Java 21 LTS for <a href="https://www.scala-sbt.org/">sbt</a> is not yet available and tracked in <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/421174">issue 421174</a>. It is not supported when <a href="https://docs.gitlab.com/ee/development/fips_compliance.html#enable-fips-mode">FIPS mode</a> is enabled. + Java 21 LTS for <a href="https://www.scala-sbt.org/">sbt</a> is limited to version 1.9.7. Support for more <a href="https://www.scala-sbt.org/">sbt</a> versions can be tracked in <a href="https://gitlab.com/gitlab-org/gitlab/-/issues/430335">issue 430335</a>. + It is not supported when <a href="https://docs.gitlab.com/ee/development/fips_compliance.html#enable-fips-mode">FIPS mode</a> is enabled. </p> </li> <li> diff --git a/doc/user/application_security/vulnerability_report/index.md b/doc/user/application_security/vulnerability_report/index.md index 577567f04e3..27f185d1cf1 100644 --- a/doc/user/application_security/vulnerability_report/index.md +++ b/doc/user/application_security/vulnerability_report/index.md @@ -86,8 +86,6 @@ The available filters are: - **Activity**: For more details, see [Activity filter](#activity-filter). The filters' criteria are combined to show only vulnerabilities matching all criteria. -An exception to this behavior is the Activity filter. For more details about how it works, see -[Activity filter](#activity-filter). <!-- vale gitlab.SubstitutionWarning = YES --> @@ -106,8 +104,6 @@ After each filter is selected: ### Tool filter -> The third-party tool filter was [introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/229661) in GitLab 13.12. - The tool filter allows you to focus on vulnerabilities detected by selected tools. When using the tool filter, you can choose: @@ -128,18 +124,22 @@ The content of the Project filter depends on the current level: ### Activity filter -The Activity filter behaves differently from the other filters. The selected values form mutually -exclusive sets to allow for precisely locating the desired vulnerability records. Additionally, not -all options can be selected in combination. - -Selection behavior when using the Activity filter: - -- **All**: Vulnerabilities with any Activity status (same as ignoring this filter). Selecting this deselects any other Activity filter options. -- **No activity**: Only vulnerabilities without either an associated issue or that are no longer detected. Selecting this deselects any other Activity filter options. -- **With issues**: Only vulnerabilities with one or more associated issues. Does not include vulnerabilities that also are no longer detected. -- **No longer detected**: Only vulnerabilities that are no longer detected in the latest pipeline scan of the `default` branch. Does not include vulnerabilities with one or more associated issues. -- **With issues** and **No longer detected**: Only vulnerabilities that have one or more associated issues and also are no longer detected in the latest pipeline scan of the `default` branch. -- **Has merge request**: Only vulnerabilities with one or more associated merge requests. +The activity filter behaves differently from the other filters. You can select only one value in +each category. + +Selection behavior when using the activity filter: + +- **Activity** + - **All activity**: Vulnerabilities with any activity status (same as ignoring this filter). Selecting this deselects all other activity filter options. +- **Detection** + - **Still detected**: Vulnerabilities that are still detected in the latest pipeline scan of the `default` branch. + - **No longer detected**: Vulnerabilities that are no longer detected in the latest pipeline scan of the `default` branch. +- **Issue** + - **Has issues**: Vulnerabilities with one or more associated issues. + - **Does not have issue**: Vulnerabilities without an associated issue. +- **Merge request** + - **Has merge request**: Vulnerabilities with one or more associated merge requests. + - **Does not have merge request**: Vulnerabilities without an associated merge request. ## View details of a vulnerability diff --git a/lib/bulk_imports/common/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb index fa09f36fdd6..723359aa438 100644 --- a/lib/bulk_imports/common/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb @@ -30,8 +30,7 @@ module BulkImports source_full_path: entity.source_full_path, pipeline_class: self.class.name, message: "Entity #{entity.status_name}", - source_version: entity.bulk_import.source_version_info.to_s, - importer: 'gitlab_migration' + source_version: entity.bulk_import.source_version_info.to_s ) ::BulkImports::FinishProjectImportWorker.perform_async(entity.project_id) if entity.project? @@ -42,7 +41,7 @@ module BulkImports attr_reader :context, :entity, :trackers def logger - @logger ||= Gitlab::Import::Logger.build + @logger ||= Logger.build end def all_other_trackers_failed? diff --git a/lib/bulk_imports/logger.rb b/lib/bulk_imports/logger.rb new file mode 100644 index 00000000000..be15c050770 --- /dev/null +++ b/lib/bulk_imports/logger.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module BulkImports + class Logger < ::Gitlab::Import::Logger + IMPORTER_NAME = 'gitlab_migration' + + def default_attributes + super.merge(importer: IMPORTER_NAME) + end + end +end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index 328c5a5941f..6037104ef03 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -159,8 +159,7 @@ module BulkImports source_full_path: context.entity.source_full_path, pipeline_class: pipeline, context_extra: context.extra, - source_version: context.entity.bulk_import.source_version_info.to_s, - importer: 'gitlab_migration' + source_version: context.entity.bulk_import.source_version_info.to_s } defaults @@ -169,7 +168,7 @@ module BulkImports end def logger - @logger ||= Gitlab::Import::Logger.build + @logger ||= Logger.build end def log_exception(exception, payload) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 0eeb0f73bba..0ee88c49fe4 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -42919,7 +42919,10 @@ msgstr "" msgid "SecurityOrchestration|Any merge request" msgstr "" -msgid "SecurityOrchestration|Apply this policy to all projects %{projectScopeType} named %{exceptionType} %{projectSelector}" +msgid "SecurityOrchestration|Apply this policy to all projects %{projectScopeType} %{exceptionType} %{projectSelector}" +msgstr "" + +msgid "SecurityOrchestration|Apply this policy to all projects %{projectScopeType} named %{frameworkSelector}" msgstr "" msgid "SecurityOrchestration|Are you sure you want to delete this policy? This action cannot be undone." @@ -43006,6 +43009,9 @@ msgstr "" msgid "SecurityOrchestration|Failed to load cluster agents." msgstr "" +msgid "SecurityOrchestration|Failed to load group projects" +msgstr "" + msgid "SecurityOrchestration|Failed to load images." msgstr "" @@ -43383,6 +43389,9 @@ msgstr "" msgid "SecurityOrchestration|by the agent named %{agents} %{cadence}%{branchExceptionsString}" msgstr "" +msgid "SecurityOrchestration|except projects" +msgstr "" + msgid "SecurityOrchestration|group level branch input" msgstr "" @@ -43440,6 +43449,9 @@ msgstr "" msgid "SecurityOrchestration|with %{exceptionType} on %{branchSelector}" msgstr "" +msgid "SecurityOrchestration|without exceptions" +msgstr "" + msgid "SecurityPolicies|Invalid or empty policy" msgstr "" diff --git a/spec/lib/bulk_imports/common/pipelines/entity_finisher_spec.rb b/spec/lib/bulk_imports/common/pipelines/entity_finisher_spec.rb index 8ca74565788..b96ea20c676 100644 --- a/spec/lib/bulk_imports/common/pipelines/entity_finisher_spec.rb +++ b/spec/lib/bulk_imports/common/pipelines/entity_finisher_spec.rb @@ -9,7 +9,7 @@ RSpec.describe BulkImports::Common::Pipelines::EntityFinisher, feature_category: context = BulkImports::Pipeline::Context.new(pipeline_tracker) subject = described_class.new(context) - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| expect(logger) .to receive(:info) .with( @@ -19,8 +19,7 @@ RSpec.describe BulkImports::Common::Pipelines::EntityFinisher, feature_category: source_full_path: entity.source_full_path, pipeline_class: described_class.name, message: 'Entity finished', - source_version: entity.bulk_import.source_version_info.to_s, - importer: 'gitlab_migration' + source_version: entity.bulk_import.source_version_info.to_s ) end diff --git a/spec/lib/bulk_imports/pipeline/runner_spec.rb b/spec/lib/bulk_imports/pipeline/runner_spec.rb index 01adde79740..2f7bdf88a60 100644 --- a/spec/lib/bulk_imports/pipeline/runner_spec.rb +++ b/spec/lib/bulk_imports/pipeline/runner_spec.rb @@ -54,7 +54,7 @@ RSpec.describe BulkImports::Pipeline::Runner, feature_category: :importers do shared_examples 'failed pipeline' do |exception_class, exception_message| it 'logs import failure' do - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| expect(logger).to receive(:error) .with( a_hash_including( @@ -69,7 +69,6 @@ RSpec.describe BulkImports::Pipeline::Runner, feature_category: :importers do 'correlation_id' => anything, 'class' => 'BulkImports::MyPipeline', 'message' => 'An object of a pipeline failed to import', - 'importer' => 'gitlab_migration', 'exception.backtrace' => anything, 'source_version' => entity.bulk_import.source_version_info.to_s ) @@ -94,14 +93,13 @@ RSpec.describe BulkImports::Pipeline::Runner, feature_category: :importers do end it 'logs a warn message and marks entity and tracker as failed' do - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| expect(logger).to receive(:warn) .with( log_params( context, message: 'Aborting entity migration due to pipeline failure', - pipeline_class: 'BulkImports::MyPipeline', - importer: 'gitlab_migration' + pipeline_class: 'BulkImports::MyPipeline' ) ) end @@ -386,7 +384,7 @@ RSpec.describe BulkImports::Pipeline::Runner, feature_category: :importers do it 'logs and returns without execution' do entity.fail_op! - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| expect(logger).to receive(:warn) .with( log_params( @@ -408,7 +406,6 @@ RSpec.describe BulkImports::Pipeline::Runner, feature_category: :importers do bulk_import_entity_type: context.entity.source_type, source_full_path: entity.source_full_path, source_version: context.entity.bulk_import.source_version_info.to_s, - importer: 'gitlab_migration', context_extra: context.extra }.merge(extra) end diff --git a/spec/requests/api/group_packages_spec.rb b/spec/requests/api/group_packages_spec.rb index 84c72edd18c..0786815c787 100644 --- a/spec/requests/api/group_packages_spec.rb +++ b/spec/requests/api/group_packages_spec.rb @@ -140,14 +140,23 @@ RSpec.describe API::GroupPackages, feature_category: :package_registry do context 'filtering on package_version' do include_context 'package filter context' - let!(:package) { create(:nuget_package, project: project, version: '2.0.4') } + let!(:package1) { create(:nuget_package, project: project, version: '2.0.4') } + let!(:package2) { create(:nuget_package, project: project) } it 'returns the versioned package' do url = group_filter_url(:version, '2.0.4') get api(url, user) expect(json_response.length).to eq(1) - expect(json_response.first['version']).to eq(package.version) + expect(json_response.first['version']).to eq(package1.version) + end + + it 'include_versionless has no effect' do + url = "/groups/#{group.id}/packages?package_version=2.0.4&include_versionless=true" + get api(url, user) + + expect(json_response.length).to eq(1) + expect(json_response.first['version']).to eq(package1.version) end end diff --git a/spec/requests/api/project_packages_spec.rb b/spec/requests/api/project_packages_spec.rb index c945db0b46d..2ac9a7d97f1 100644 --- a/spec/requests/api/project_packages_spec.rb +++ b/spec/requests/api/project_packages_spec.rb @@ -207,6 +207,14 @@ RSpec.describe API::ProjectPackages, feature_category: :package_registry do expect(json_response.length).to eq(1) expect(json_response.first['version']).to eq(package2.version) end + + it 'include_versionless has no effect' do + url = "/projects/#{project.id}/packages?package_version=2.0.4&include_versionless=true" + get api(url, user) + + expect(json_response.length).to eq(1) + expect(json_response.first['version']).to eq(package2.version) + end end it_behaves_like 'with versionless packages' diff --git a/spec/services/bulk_imports/file_download_service_spec.rb b/spec/services/bulk_imports/file_download_service_spec.rb index 1734ea45507..8b9304065de 100644 --- a/spec/services/bulk_imports/file_download_service_spec.rb +++ b/spec/services/bulk_imports/file_download_service_spec.rb @@ -82,18 +82,17 @@ RSpec.describe BulkImports::FileDownloadService, feature_category: :importers do context 'when content-type is not valid' do let(:content_type) { 'invalid' } - let(:import_logger) { instance_double(Gitlab::Import::Logger) } + let(:import_logger) { instance_double(BulkImports::Logger) } before do - allow(Gitlab::Import::Logger).to receive(:build).and_return(import_logger) + allow(BulkImports::Logger).to receive(:build).and_return(import_logger) allow(import_logger).to receive(:warn) end it 'logs and raises an error' do expect(import_logger).to receive(:warn).once.with( message: 'Invalid content type', - response_headers: headers, - importer: 'gitlab_migration' + response_headers: headers ) expect { subject.execute }.to raise_error(described_class::ServiceError, 'Invalid content type') diff --git a/spec/services/bulk_imports/process_service_spec.rb b/spec/services/bulk_imports/process_service_spec.rb index 9d519bc3b05..f5566819039 100644 --- a/spec/services/bulk_imports/process_service_spec.rb +++ b/spec/services/bulk_imports/process_service_spec.rb @@ -204,14 +204,13 @@ RSpec.describe BulkImports::ProcessService, feature_category: :importers do end it 'logs an info message for the skipped pipelines' do - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| expect(logger).to receive(:info).with( message: 'Pipeline skipped as source instance version not compatible with pipeline', bulk_import_entity_id: entity.id, bulk_import_id: entity.bulk_import_id, bulk_import_entity_type: entity.source_type, source_full_path: entity.source_full_path, - importer: 'gitlab_migration', pipeline_class: 'PipelineClass4', minimum_source_version: '15.1.0', maximum_source_version: nil, @@ -224,7 +223,6 @@ RSpec.describe BulkImports::ProcessService, feature_category: :importers do bulk_import_id: entity.bulk_import_id, bulk_import_entity_type: entity.source_type, source_full_path: entity.source_full_path, - importer: 'gitlab_migration', pipeline_class: 'PipelineClass5', minimum_source_version: '16.0.0', maximum_source_version: nil, diff --git a/spec/support/shared_examples/graphql/resolvers/packages_resolvers_shared_examples.rb b/spec/support/shared_examples/graphql/resolvers/packages_resolvers_shared_examples.rb index 1ea3074b3cb..98eadc507d7 100644 --- a/spec/support/shared_examples/graphql/resolvers/packages_resolvers_shared_examples.rb +++ b/spec/support/shared_examples/graphql/resolvers/packages_resolvers_shared_examples.rb @@ -56,6 +56,12 @@ RSpec.shared_examples 'group and projects packages resolver' do let(:args) { { package_version: '1.0.0', sort: 'CREATED_DESC' } } it { is_expected.to eq([conan_package]) } + + it 'includes_versionless has no effect' do + args[:include_versionless] = true + + is_expected.to eq([conan_package]) + end end context 'filter by status' do diff --git a/spec/workers/bulk_imports/entity_worker_spec.rb b/spec/workers/bulk_imports/entity_worker_spec.rb index 507e6773bdb..690555aa08f 100644 --- a/spec/workers/bulk_imports/entity_worker_spec.rb +++ b/spec/workers/bulk_imports/entity_worker_spec.rb @@ -34,7 +34,7 @@ RSpec.describe BulkImports::EntityWorker, feature_category: :importers do end it 'enqueues the pipeline workers of the first stage and then re-enqueues itself' do - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| expect(logger).to receive(:info).with(hash_including('message' => 'Stage starting', 'entity_stage' => 0)) expect(logger).to receive(:info).with(hash_including('message' => 'Stage running', 'entity_stage' => 0)) end @@ -59,7 +59,7 @@ RSpec.describe BulkImports::EntityWorker, feature_category: :importers do end it 'does not enqueue the pipeline workers from the next stage and re-enqueues itself' do - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| expect(logger).to receive(:info).with(hash_including('message' => 'Stage running', 'entity_stage' => 0)) end @@ -76,7 +76,7 @@ RSpec.describe BulkImports::EntityWorker, feature_category: :importers do end it 'enqueues the pipeline workers from the next stage and re-enqueues itself' do - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| expect(logger).to receive(:info).with(hash_including('message' => 'Stage starting', 'entity_stage' => 1)) end diff --git a/spec/workers/bulk_imports/export_request_worker_spec.rb b/spec/workers/bulk_imports/export_request_worker_spec.rb index 0acc44c5cbf..e9d0b6b24b2 100644 --- a/spec/workers/bulk_imports/export_request_worker_spec.rb +++ b/spec/workers/bulk_imports/export_request_worker_spec.rb @@ -71,7 +71,7 @@ RSpec.describe BulkImports::ExportRequestWorker, feature_category: :importers do entity.update!(source_xid: nil) - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| expect(logger).to receive(:error).with( a_hash_including( 'bulk_import_entity_id' => entity.id, @@ -82,7 +82,6 @@ RSpec.describe BulkImports::ExportRequestWorker, feature_category: :importers do 'exception.class' => 'NoMethodError', 'exception.message' => /^undefined method `model_id' for nil:NilClass/, 'message' => 'Failed to fetch source entity id', - 'importer' => 'gitlab_migration', 'source_version' => entity.bulk_import.source_version_info.to_s ) ).twice diff --git a/spec/workers/bulk_imports/finish_batched_pipeline_worker_spec.rb b/spec/workers/bulk_imports/finish_batched_pipeline_worker_spec.rb index 8c67583f6b5..610138c2723 100644 --- a/spec/workers/bulk_imports/finish_batched_pipeline_worker_spec.rb +++ b/spec/workers/bulk_imports/finish_batched_pipeline_worker_spec.rb @@ -15,6 +15,12 @@ RSpec.describe BulkImports::FinishBatchedPipelineWorker, feature_category: :impo describe '#perform' do context 'when import is in progress' do it 'marks the pipeline as finished' do + expect_next_instance_of(BulkImports::Logger) do |logger| + expect(logger).to receive(:info).with( + a_hash_including('message' => 'Tracker finished') + ) + end + expect { subject.perform(pipeline_tracker.id) } .to change { pipeline_tracker.reload.finished? } .from(false).to(true) @@ -47,6 +53,12 @@ RSpec.describe BulkImports::FinishBatchedPipelineWorker, feature_category: :impo it 'fails pipeline tracker and its batches' do create(:bulk_import_batch_tracker, :finished, tracker: pipeline_tracker) + expect_next_instance_of(BulkImports::Logger) do |logger| + expect(logger).to receive(:error).with( + a_hash_including('message' => 'Tracker stale. Failing batches and tracker') + ) + end + subject.perform(pipeline_tracker.id) expect(pipeline_tracker.reload.failed?).to eq(true) diff --git a/spec/workers/bulk_imports/pipeline_batch_worker_spec.rb b/spec/workers/bulk_imports/pipeline_batch_worker_spec.rb index c4cd2ca2384..9ac297ae757 100644 --- a/spec/workers/bulk_imports/pipeline_batch_worker_spec.rb +++ b/spec/workers/bulk_imports/pipeline_batch_worker_spec.rb @@ -65,6 +65,10 @@ RSpec.describe BulkImports::PipelineBatchWorker, feature_category: :importers do describe '#perform' do it 'runs the given pipeline batch successfully' do expect(BulkImports::FinishBatchedPipelineWorker).to receive(:perform_async).with(tracker.id) + expect_next_instance_of(BulkImports::Logger) do |logger| + expect(logger).to receive(:info).with(a_hash_including('message' => 'Batch tracker started')) + expect(logger).to receive(:info).with(a_hash_including('message' => 'Batch tracker finished')) + end worker.perform(batch.id) @@ -173,10 +177,12 @@ RSpec.describe BulkImports::PipelineBatchWorker, feature_category: :importers do expect(Gitlab::ErrorTracking).to receive(:track_exception).with( instance_of(StandardError), hash_including( - batch_id: batch.id, - tracker_id: tracker.id, - pipeline_class: 'FakePipeline', - pipeline_step: 'pipeline_batch_worker_run' + 'message' => 'Batch tracker failed', + 'batch_id' => batch.id, + 'tracker_id' => tracker.id, + 'pipeline_class' => 'FakePipeline', + 'pipeline_step' => 'pipeline_batch_worker_run', + 'importer' => 'gitlab_migration' ) ) diff --git a/spec/workers/bulk_imports/pipeline_worker_spec.rb b/spec/workers/bulk_imports/pipeline_worker_spec.rb index 358cea0110d..6ea7334f6a6 100644 --- a/spec/workers/bulk_imports/pipeline_worker_spec.rb +++ b/spec/workers/bulk_imports/pipeline_worker_spec.rb @@ -56,7 +56,7 @@ RSpec.describe BulkImports::PipelineWorker, feature_category: :importers do end it 'runs the given pipeline successfully' do - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| expect(logger) .to receive(:info) .with( @@ -93,7 +93,7 @@ RSpec.describe BulkImports::PipelineWorker, feature_category: :importers do it 'logs and sets status as failed' do job = { 'args' => [pipeline_tracker.id, pipeline_tracker.stage, entity.id] } - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| expect(logger) .to receive(:error) .with( @@ -216,7 +216,7 @@ RSpec.describe BulkImports::PipelineWorker, feature_category: :importers do entity.update!(status: -1) - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| allow(logger).to receive(:info) expect(logger) @@ -268,7 +268,7 @@ RSpec.describe BulkImports::PipelineWorker, feature_category: :importers do expect(tracker).to receive(:retry).and_call_original end - expect_next_instance_of(Gitlab::Import::Logger) do |logger| + expect_next_instance_of(BulkImports::Logger) do |logger| expect(logger) .to receive(:info) .with( diff --git a/spec/workers/bulk_imports/stuck_import_worker_spec.rb b/spec/workers/bulk_imports/stuck_import_worker_spec.rb index ba1b1b66b00..eadf3864190 100644 --- a/spec/workers/bulk_imports/stuck_import_worker_spec.rb +++ b/spec/workers/bulk_imports/stuck_import_worker_spec.rb @@ -9,17 +9,46 @@ RSpec.describe BulkImports::StuckImportWorker, feature_category: :importers do let_it_be(:stale_started_bulk_import) { create(:bulk_import, :started, created_at: 3.days.ago) } let_it_be(:stale_created_bulk_import_entity) { create(:bulk_import_entity, :created, created_at: 3.days.ago) } let_it_be(:stale_started_bulk_import_entity) { create(:bulk_import_entity, :started, created_at: 3.days.ago) } - let_it_be(:started_bulk_import_tracker) { create(:bulk_import_tracker, :started, entity: stale_started_bulk_import_entity) } + + let_it_be(:started_bulk_import_tracker) do + create(:bulk_import_tracker, :started, entity: stale_started_bulk_import_entity) + end subject { described_class.new.perform } describe 'perform' do it 'updates the status of bulk imports to timeout' do + expect_next_instance_of(BulkImports::Logger) do |logger| + allow(logger).to receive(:error) + expect(logger).to receive(:error).with( + message: 'BulkImport stale', + bulk_import_id: stale_created_bulk_import.id + ) + expect(logger).to receive(:error).with( + message: 'BulkImport stale', + bulk_import_id: stale_started_bulk_import.id + ) + end + expect { subject }.to change { stale_created_bulk_import.reload.status_name }.from(:created).to(:timeout) .and change { stale_started_bulk_import.reload.status_name }.from(:started).to(:timeout) end it 'updates the status of bulk import entities to timeout' do + expect_next_instance_of(BulkImports::Logger) do |logger| + allow(logger).to receive(:error) + expect(logger).to receive(:error).with( + message: 'BulkImports::Entity stale', + bulk_import_entity_id: stale_created_bulk_import_entity.id, + bulk_import_id: stale_created_bulk_import_entity.bulk_import_id + ) + expect(logger).to receive(:error).with( + message: 'BulkImports::Entity stale', + bulk_import_entity_id: stale_started_bulk_import_entity.id, + bulk_import_id: stale_started_bulk_import_entity.bulk_import_id + ) + end + expect { subject }.to change { stale_created_bulk_import_entity.reload.status_name }.from(:created).to(:timeout) .and change { stale_started_bulk_import_entity.reload.status_name }.from(:started).to(:timeout) end |