Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2023-11-14 11:41:52 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2023-11-14 11:41:52 +0300
commit585826cb22ecea5998a2c2a4675735c94bdeedac (patch)
tree5b05f0b30d33cef48963609e8a18a4dff260eab3 /lib/gitlab
parentdf221d036e5d0c6c0ee4d55b9c97f481ee05dee8 (diff)
Add latest changes from gitlab-org/gitlab@16-6-stable-eev16.6.0-rc42
Diffstat (limited to 'lib/gitlab')
-rw-r--r--lib/gitlab/analytics/cycle_analytics/request_params.rb3
-rw-r--r--lib/gitlab/application_rate_limiter.rb1
-rw-r--r--lib/gitlab/auth.rb24
-rw-r--r--lib/gitlab/auth/saml/config.rb15
-rw-r--r--lib/gitlab/auth/two_factor_auth_verifier.rb2
-rw-r--r--lib/gitlab/background_migration/backfill_packages_tags_project_id.rb30
-rw-r--r--lib/gitlab/background_migration/batched_migration_job.rb2
-rw-r--r--lib/gitlab/background_migration/delete_invalid_protected_branch_merge_access_levels.rb28
-rw-r--r--lib/gitlab/background_migration/delete_invalid_protected_branch_push_access_levels.rb28
-rw-r--r--lib/gitlab/background_migration/delete_invalid_protected_tag_create_access_levels.rb28
-rw-r--r--lib/gitlab/base_doorkeeper_controller.rb4
-rw-r--r--lib/gitlab/bitbucket_import/importers/issue_importer.rb2
-rw-r--r--lib/gitlab/bitbucket_import/importers/issues_importer.rb18
-rw-r--r--lib/gitlab/bitbucket_import/importers/pull_request_importer.rb2
-rw-r--r--lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb124
-rw-r--r--lib/gitlab/bitbucket_import/importers/repository_importer.rb11
-rw-r--r--lib/gitlab/bitbucket_import/loggable.rb4
-rw-r--r--lib/gitlab/checks/diff_check.rb3
-rw-r--r--lib/gitlab/checks/file_size_check/any_oversized_blobs.rb2
-rw-r--r--lib/gitlab/ci/ansi2json/line.rb9
-rw-r--r--lib/gitlab/ci/ansi2json/state.rb1
-rw-r--r--lib/gitlab/ci/build/context/build.rb10
-rw-r--r--lib/gitlab/ci/components/instance_path.rb4
-rw-r--r--lib/gitlab/ci/config/entry/job.rb23
-rw-r--r--lib/gitlab/ci/config/entry/pages.rb31
-rw-r--r--lib/gitlab/ci/config/entry/processable.rb2
-rw-r--r--lib/gitlab/ci/config/header/input.rb11
-rw-r--r--lib/gitlab/ci/config/interpolation/inputs/base_input.rb46
-rw-r--r--lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb9
-rw-r--r--lib/gitlab/ci/config/interpolation/inputs/number_input.rb17
-rw-r--r--lib/gitlab/ci/config/interpolation/inputs/string_input.rb28
-rw-r--r--lib/gitlab/ci/jwt_v2.rb30
-rw-r--r--lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb7
-rw-r--r--lib/gitlab/ci/parsers/sbom/source/base_source.rb46
-rw-r--r--lib/gitlab/ci/parsers/sbom/source/container_scanning.rb41
-rw-r--r--lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb27
-rw-r--r--lib/gitlab/ci/parsers/security/common.rb2
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schema_validator.rb16
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/cluster-image-scanning-report-format.json1085
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/container-scanning-report-format.json1017
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/coverage-fuzzing-report-format.json975
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dast-report-format.json1380
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dependency-scanning-report-format.json1083
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/sast-report-format.json970
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/secret-detection-report-format.json994
-rw-r--r--lib/gitlab/ci/status/build/cancelable.rb2
-rw-r--r--lib/gitlab/ci/status/composite.rb2
-rw-r--r--lib/gitlab/ci/status/waiting_for_callback.rb33
-rw-r--r--lib/gitlab/ci/templates/Cosign.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml23
-rw-r--r--lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml6
-rw-r--r--lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml3
-rw-r--r--lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml3
-rw-r--r--lib/gitlab/ci/yaml_processor/dag.rb6
-rw-r--r--lib/gitlab/ci/yaml_processor/result.rb6
-rw-r--r--lib/gitlab/composer/version_index.rb3
-rw-r--r--lib/gitlab/config/entry/validators.rb3
-rw-r--r--lib/gitlab/config_checker/puma_rugged_checker.rb28
-rw-r--r--lib/gitlab/database/dictionary.rb60
-rw-r--r--lib/gitlab/database/dynamic_model_helpers.rb3
-rw-r--r--lib/gitlab/database/gitlab_schema.rb27
-rw-r--r--lib/gitlab/database/migration.rb6
-rw-r--r--lib/gitlab/database/migration_helpers.rb6
-rw-r--r--lib/gitlab/database/migration_helpers/convert_to_bigint.rb88
-rw-r--r--lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb2
-rw-r--r--lib/gitlab/database/migrations/batched_background_migration_helpers.rb2
-rw-r--r--lib/gitlab/database/migrations/milestone_mixin.rb5
-rw-r--r--lib/gitlab/database/partitioning/partition_manager.rb19
-rw-r--r--lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb117
-rw-r--r--lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb2
-rw-r--r--lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb47
-rw-r--r--lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/columns.rb91
-rw-r--r--lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/common_table_expressions.rb74
-rw-r--r--lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/froms.rb68
-rw-r--r--lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/node.rb118
-rw-r--r--lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/references.rb64
-rw-r--r--lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/select_stmt.rb81
-rw-r--r--lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/targets.rb97
-rw-r--r--lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/type.rb16
-rw-r--r--lib/gitlab/database/schema_cache_with_renamed_table.rb22
-rw-r--r--lib/gitlab/database/schema_cache_with_renamed_table_legacy.rb55
-rw-r--r--lib/gitlab/database/tables_locker.rb2
-rw-r--r--lib/gitlab/database/tables_truncate.rb2
-rw-r--r--lib/gitlab/discussions_diff/file_collection.rb14
-rw-r--r--lib/gitlab/email/handler/service_desk_handler.rb30
-rw-r--r--lib/gitlab/emoji.rb2
-rw-r--r--lib/gitlab/encrypted_command_base.rb14
-rw-r--r--lib/gitlab/encrypted_configuration.rb2
-rw-r--r--lib/gitlab/encrypted_ldap_command.rb2
-rw-r--r--lib/gitlab/encrypted_redis_command.rb56
-rw-r--r--lib/gitlab/file_detector.rb2
-rw-r--r--lib/gitlab/git/blame.rb7
-rw-r--r--lib/gitlab/git/blob.rb2
-rw-r--r--lib/gitlab/git/commit.rb3
-rw-r--r--lib/gitlab/git/ref.rb1
-rw-r--r--lib/gitlab/git/repository.rb1
-rw-r--r--lib/gitlab/git/rugged_impl/blob.rb107
-rw-r--r--lib/gitlab/git/rugged_impl/commit.rb115
-rw-r--r--lib/gitlab/git/rugged_impl/ref.rb20
-rw-r--r--lib/gitlab/git/rugged_impl/repository.rb79
-rw-r--r--lib/gitlab/git/rugged_impl/tree.rb147
-rw-r--r--lib/gitlab/git/rugged_impl/use_rugged.rb50
-rw-r--r--lib/gitlab/git/tree.rb5
-rw-r--r--lib/gitlab/git_access.rb4
-rw-r--r--lib/gitlab/git_access_project.rb2
-rw-r--r--lib/gitlab/git_audit_event.rb28
-rw-r--r--lib/gitlab/gitaly_client.rb14
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb9
-rw-r--r--lib/gitlab/gitaly_client/conflict_files_stitcher.rb10
-rw-r--r--lib/gitlab/gitaly_client/repository_service.rb7
-rw-r--r--lib/gitlab/gitaly_client/storage_settings.rb8
-rw-r--r--lib/gitlab/github_import/attachments_downloader.rb24
-rw-r--r--lib/gitlab/github_import/client.rb10
-rw-r--r--lib/gitlab/github_import/issuable_finder.rb14
-rw-r--r--lib/gitlab/github_import/job_delay_calculator.rb2
-rw-r--r--lib/gitlab/github_import/label_finder.rb16
-rw-r--r--lib/gitlab/github_import/milestone_finder.rb18
-rw-r--r--lib/gitlab/github_import/object_counter.rb2
-rw-r--r--lib/gitlab/github_import/parallel_scheduling.rb18
-rw-r--r--lib/gitlab/github_import/representation/to_hash.rb4
-rw-r--r--lib/gitlab/gon_helper.rb3
-rw-r--r--lib/gitlab/graphql/tracers/timer_tracer.rb4
-rw-r--r--lib/gitlab/group_search_results.rb2
-rw-r--r--lib/gitlab/identifier.rb9
-rw-r--r--lib/gitlab/import_export/command_line_util.rb21
-rw-r--r--lib/gitlab/import_export/error.rb4
-rw-r--r--lib/gitlab/import_export/project/sample/date_calculator.rb2
-rw-r--r--lib/gitlab/instrumentation/redis_base.rb6
-rw-r--r--lib/gitlab/instrumentation/redis_interceptor.rb4
-rw-r--r--lib/gitlab/instrumentation_helper.rb10
-rw-r--r--lib/gitlab/internal_events.rb2
-rw-r--r--lib/gitlab/issues/rebalancing/state.rb2
-rw-r--r--lib/gitlab/jira/http_client.rb15
-rw-r--r--lib/gitlab/jira/middleware.rb23
-rw-r--r--lib/gitlab/jira_import/base_importer.rb4
-rw-r--r--lib/gitlab/jira_import/issues_importer.rb2
-rw-r--r--lib/gitlab/job_waiter.rb29
-rw-r--r--lib/gitlab/kubernetes/kubeconfig/template.rb2
-rw-r--r--lib/gitlab/legacy_http.rb4
-rw-r--r--lib/gitlab/memory/reporter.rb8
-rw-r--r--lib/gitlab/memory/reports_uploader.rb4
-rw-r--r--lib/gitlab/merge_requests/mergeability/check_result.rb5
-rw-r--r--lib/gitlab/metrics/exporter/metrics_middleware.rb8
-rw-r--r--lib/gitlab/middleware/go.rb2
-rw-r--r--lib/gitlab/middleware/path_traversal_check.rb45
-rw-r--r--lib/gitlab/omniauth_initializer.rb11
-rw-r--r--lib/gitlab/optimistic_locking.rb4
-rw-r--r--lib/gitlab/pages/deployment_update.rb15
-rw-r--r--lib/gitlab/pagination/cursor_based_keyset.rb6
-rw-r--r--lib/gitlab/patch/sidekiq_cron_poller.rb2
-rw-r--r--lib/gitlab/patch/sidekiq_scheduled_enq.rb7
-rw-r--r--lib/gitlab/project_template.rb7
-rw-r--r--lib/gitlab/push_options.rb1
-rw-r--r--lib/gitlab/quick_actions/merge_request_actions.rb23
-rw-r--r--lib/gitlab/redis.rb1
-rw-r--r--lib/gitlab/redis/cluster_util.rb9
-rw-r--r--lib/gitlab/redis/multi_store.rb47
-rw-r--r--lib/gitlab/redis/pubsub.rb13
-rw-r--r--lib/gitlab/redis/shared_state.rb6
-rw-r--r--lib/gitlab/redis/wrapper.rb40
-rw-r--r--lib/gitlab/regex.rb1
-rw-r--r--lib/gitlab/regex/packages.rb8
-rw-r--r--lib/gitlab/regex/packages/protection/rules.rb15
-rw-r--r--lib/gitlab/request_forgery_protection.rb4
-rw-r--r--lib/gitlab/rugged_instrumentation.rb45
-rw-r--r--lib/gitlab/search_results.rb8
-rw-r--r--lib/gitlab/seeders/ci/catalog/resource_seeder.rb114
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb11
-rw-r--r--lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb2
-rw-r--r--lib/gitlab/sidekiq_middleware/server_metrics.rb100
-rw-r--r--lib/gitlab/sidekiq_middleware/skip_jobs.rb13
-rw-r--r--lib/gitlab/sidekiq_status.rb13
-rw-r--r--lib/gitlab/tracking.rb3
-rw-r--r--lib/gitlab/url_builder.rb2
-rw-r--r--lib/gitlab/usage/metric_definition.rb23
-rw-r--r--lib/gitlab/usage_data.rb1
-rw-r--r--lib/gitlab/usage_data_counters/hll_redis_counter.rb5
-rw-r--r--lib/gitlab/usage_data_counters/redis_counter.rb7
-rw-r--r--lib/gitlab/workhorse.rb3
184 files changed, 9940 insertions, 1100 deletions
diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb
index cea25ba2db4..0c4a0afa1d5 100644
--- a/lib/gitlab/analytics/cycle_analytics/request_params.rb
+++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb
@@ -203,7 +203,8 @@ module Gitlab
def validate_date_range
return if created_after.nil? || created_before.nil?
- if (created_before - created_after) > MAX_RANGE_DAYS
+ time_period = created_before.at_beginning_of_day - created_after.at_beginning_of_day
+ if time_period > MAX_RANGE_DAYS
errors.add(:created_after, s_('CycleAnalytics|The given date range is larger than 180 days'))
end
end
diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb
index bf3f5b61825..469927b8a53 100644
--- a/lib/gitlab/application_rate_limiter.rb
+++ b/lib/gitlab/application_rate_limiter.rb
@@ -55,6 +55,7 @@ module Gitlab
phone_verification_send_code: { threshold: 10, interval: 1.hour },
phone_verification_verify_code: { threshold: 10, interval: 10.minutes },
namespace_exists: { threshold: 20, interval: 1.minute },
+ update_namespace_name: { threshold: -> { application_settings.update_namespace_name_rate_limit }, interval: 1.hour },
fetch_google_ip_list: { threshold: 10, interval: 1.minute },
project_fork_sync: { threshold: 10, interval: 30.minutes },
ai_action: { threshold: 160, interval: 8.hours },
diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb
index fc1f7a1583c..578cfb52714 100644
--- a/lib/gitlab/auth.rb
+++ b/lib/gitlab/auth.rb
@@ -60,10 +60,10 @@ module Gitlab
Gitlab.config.omniauth.enabled
end
- def find_for_git_client(login, password, project:, ip:)
- raise "Must provide an IP for rate limiting" if ip.nil?
+ def find_for_git_client(login, password, project:, request:)
+ raise "Must provide an IP for rate limiting" if request.ip.nil?
- rate_limiter = Gitlab::Auth::IpRateLimiter.new(ip)
+ rate_limiter = Gitlab::Auth::IpRateLimiter.new(request.ip)
raise IpBlocked if !skip_rate_limit?(login: login) && rate_limiter.banned?
@@ -80,7 +80,7 @@ module Gitlab
user_with_password_for_git(login, password) ||
Gitlab::Auth::Result::EMPTY
- rate_limit!(rate_limiter, success: result.success?, login: login)
+ rate_limit!(rate_limiter, success: result.success?, login: login, request: request)
look_to_limit_user(result.actor)
return result if result.success? || authenticate_using_internal_or_ldap_password?
@@ -142,7 +142,7 @@ module Gitlab
private
- def rate_limit!(rate_limiter, success:, login:)
+ def rate_limit!(rate_limiter, success:, login:, request:)
return if skip_rate_limit?(login: login)
if success
@@ -155,8 +155,18 @@ module Gitlab
# request from this IP if needed.
# This returns true when the failures are over the threshold and the IP
# is banned.
- Gitlab::AppLogger.info "IP #{rate_limiter.ip} failed to login " \
- "as #{login} but has been temporarily banned from Git auth"
+
+ message = "Rack_Attack: Git auth failures has exceeded the threshold. " \
+ "IP has been temporarily banned from Git auth."
+
+ Gitlab::AuthLogger.error(
+ message: message,
+ env: :blocklist,
+ remote_ip: request.ip,
+ request_method: request.request_method,
+ path: request.fullpath,
+ login: login
+ )
end
end
diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb
index 7524d8b9f85..e6c9f04eff5 100644
--- a/lib/gitlab/auth/saml/config.rb
+++ b/lib/gitlab/auth/saml/config.rb
@@ -8,6 +8,21 @@ module Gitlab
def enabled?
::AuthHelper.saml_providers.any?
end
+
+ def default_attribute_statements
+ defaults = OmniAuth::Strategies::SAML.default_options[:attribute_statements].to_hash.deep_symbolize_keys
+ defaults[:nickname] = %w[username nickname]
+ defaults[:name] << 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name'
+ defaults[:name] << 'http://schemas.microsoft.com/ws/2008/06/identity/claims/name'
+ defaults[:email] << 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress'
+ defaults[:email] << 'http://schemas.microsoft.com/ws/2008/06/identity/claims/emailaddress'
+ defaults[:first_name] << 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname'
+ defaults[:first_name] << 'http://schemas.microsoft.com/ws/2008/06/identity/claims/givenname'
+ defaults[:last_name] << 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname'
+ defaults[:last_name] << 'http://schemas.microsoft.com/ws/2008/06/identity/claims/surname'
+
+ defaults
+ end
end
DEFAULT_PROVIDER_NAME = 'saml'
diff --git a/lib/gitlab/auth/two_factor_auth_verifier.rb b/lib/gitlab/auth/two_factor_auth_verifier.rb
index fbdfd105ee3..4b66aaf0e6a 100644
--- a/lib/gitlab/auth/two_factor_auth_verifier.rb
+++ b/lib/gitlab/auth/two_factor_auth_verifier.rb
@@ -36,7 +36,7 @@ module Gitlab
return false unless time
- two_factor_grace_period.hours.since(time) < Time.current
+ two_factor_grace_period.hours.since(time).past?
end
def allow_2fa_bypass_for_provider
diff --git a/lib/gitlab/background_migration/backfill_packages_tags_project_id.rb b/lib/gitlab/background_migration/backfill_packages_tags_project_id.rb
new file mode 100644
index 00000000000..04fd09f81f0
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_packages_tags_project_id.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # This migration populates the new `packages_tags.project_id` column from joining with `packages_packages` table
+ class BackfillPackagesTagsProjectId < BatchedMigrationJob
+ operation_name :update_all # This is used as the key on collecting metrics
+ scope_to ->(relation) { relation.where(project_id: nil) }
+ feature_category :package_registry
+
+ def perform
+ each_sub_batch do |sub_batch|
+ joined = sub_batch
+ .joins('INNER JOIN packages_packages ON packages_tags.package_id = packages_packages.id')
+ .select('packages_tags.id, packages_packages.project_id')
+
+ ApplicationRecord.connection.execute <<~SQL
+ WITH joined_cte(packages_tag_id, project_id) AS MATERIALIZED (
+ #{joined.to_sql}
+ )
+ UPDATE packages_tags
+ SET project_id = joined_cte.project_id
+ FROM joined_cte
+ WHERE id = joined_cte.packages_tag_id
+ SQL
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb
index 952e6d01f1a..9e9fc9b98b7 100644
--- a/lib/gitlab/background_migration/batched_migration_job.rb
+++ b/lib/gitlab/background_migration/batched_migration_job.rb
@@ -130,7 +130,7 @@ module Gitlab
end
def base_relation
- define_batchable_model(batch_table, connection: connection)
+ define_batchable_model(batch_table, connection: connection, primary_key: batch_column)
.where(batch_column => start_id..end_id)
end
diff --git a/lib/gitlab/background_migration/delete_invalid_protected_branch_merge_access_levels.rb b/lib/gitlab/background_migration/delete_invalid_protected_branch_merge_access_levels.rb
new file mode 100644
index 00000000000..99bc638532a
--- /dev/null
+++ b/lib/gitlab/background_migration/delete_invalid_protected_branch_merge_access_levels.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # A job to remove protected_branch_merge_access_levels for groups that do not have project_group_links
+ # to the project for the associated protected branch
+ class DeleteInvalidProtectedBranchMergeAccessLevels < BatchedMigrationJob
+ operation_name :delete_invalid_protected_branch_merge_access_levels
+ scope_to ->(relation) { relation.where.not(group_id: nil) }
+ feature_category :source_code_management
+
+ def perform
+ each_sub_batch do |sub_batch|
+ sub_batch
+ .joins('INNER JOIN protected_branches ON protected_branches.id = protected_branch_id')
+ .joins(%(
+ LEFT OUTER JOIN project_group_links pgl
+ ON pgl.group_id = protected_branch_merge_access_levels.group_id
+ AND pgl.project_id = protected_branches.project_id
+ ))
+ .where(%(
+ pgl.id IS NULL
+ )).delete_all
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/delete_invalid_protected_branch_push_access_levels.rb b/lib/gitlab/background_migration/delete_invalid_protected_branch_push_access_levels.rb
new file mode 100644
index 00000000000..a6934cf5adc
--- /dev/null
+++ b/lib/gitlab/background_migration/delete_invalid_protected_branch_push_access_levels.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # A job to remove protected_branch_push_access_levels for groups that do not have project_group_links
+ # to the project for the associated protected branch
+ class DeleteInvalidProtectedBranchPushAccessLevels < BatchedMigrationJob
+ operation_name :delete_invalid_protected_branch_push_access_levels
+ scope_to ->(relation) { relation.where.not(group_id: nil) }
+ feature_category :source_code_management
+
+ def perform
+ each_sub_batch do |sub_batch|
+ sub_batch
+ .joins('INNER JOIN protected_branches ON protected_branches.id = protected_branch_id')
+ .joins(%(
+ LEFT OUTER JOIN project_group_links pgl
+ ON pgl.group_id = protected_branch_push_access_levels.group_id
+ AND pgl.project_id = protected_branches.project_id
+ ))
+ .where(%(
+ pgl.id IS NULL
+ )).delete_all
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/background_migration/delete_invalid_protected_tag_create_access_levels.rb b/lib/gitlab/background_migration/delete_invalid_protected_tag_create_access_levels.rb
new file mode 100644
index 00000000000..8c59e42a9f6
--- /dev/null
+++ b/lib/gitlab/background_migration/delete_invalid_protected_tag_create_access_levels.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # A job to remove protected_tag_create_access_levels for groups that do not have project_group_links
+ # to the project for the associated protected branch
+ class DeleteInvalidProtectedTagCreateAccessLevels < BatchedMigrationJob
+ operation_name :delete_invalid_protected_tag_create_access_levels
+ scope_to ->(relation) { relation.where.not(group_id: nil) }
+ feature_category :source_code_management
+
+ def perform
+ each_sub_batch do |sub_batch|
+ sub_batch
+ .joins('INNER JOIN protected_tags ON protected_tags.id = protected_tag_id')
+ .joins(%(
+ LEFT OUTER JOIN project_group_links pgl
+ ON pgl.group_id = protected_tag_create_access_levels.group_id
+ AND pgl.project_id = protected_tags.project_id
+ ))
+ .where(%(
+ pgl.id IS NULL
+ )).delete_all
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb
index 91994c2fa95..c8520993b8e 100644
--- a/lib/gitlab/base_doorkeeper_controller.rb
+++ b/lib/gitlab/base_doorkeeper_controller.rb
@@ -3,7 +3,8 @@
# This is a base controller for doorkeeper.
# It adds the `can?` helper used in the views.
module Gitlab
- class BaseDoorkeeperController < BaseActionController
+ # rubocop:disable Rails/ApplicationController
+ class BaseDoorkeeperController < ActionController::Base
include Gitlab::Allowable
include EnforcesTwoFactorAuthentication
include SessionsHelper
@@ -12,4 +13,5 @@ module Gitlab
helper_method :can?
end
+ # rubocop:enable Rails/ApplicationController
end
diff --git a/lib/gitlab/bitbucket_import/importers/issue_importer.rb b/lib/gitlab/bitbucket_import/importers/issue_importer.rb
index 2c3be67eabc..d194a311278 100644
--- a/lib/gitlab/bitbucket_import/importers/issue_importer.rb
+++ b/lib/gitlab/bitbucket_import/importers/issue_importer.rb
@@ -40,6 +40,8 @@ module Gitlab
project.issues.create!(attributes)
+ metrics.issues_counter.increment
+
log_info(import_stage: 'import_issue', message: 'finished', iid: object[:iid])
rescue StandardError => e
track_import_failure!(project, exception: e)
diff --git a/lib/gitlab/bitbucket_import/importers/issues_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_importer.rb
index 6162433e701..8ab82ddb0be 100644
--- a/lib/gitlab/bitbucket_import/importers/issues_importer.rb
+++ b/lib/gitlab/bitbucket_import/importers/issues_importer.rb
@@ -7,17 +7,21 @@ module Gitlab
include ParallelScheduling
def execute
+ return job_waiter unless repo.issues_enabled?
+
log_info(import_stage: 'import_issues', message: 'importing issues')
issues = client.issues(project.import_source)
labels = build_labels_hash
- issues.each do |issue|
+ issues.each_with_index do |issue, index|
job_waiter.jobs_remaining += 1
next if already_enqueued?(issue)
+ allocate_issues_internal_id! if index == 0
+
job_delay = calculate_job_delay(job_waiter.jobs_remaining)
issue_hash = issue.to_hash.merge({ issue_type_id: default_issue_type_id, label_id: labels[issue.kind] })
@@ -49,11 +53,23 @@ module Gitlab
::WorkItems::Type.default_issue_type.id
end
+ def allocate_issues_internal_id!
+ last_bitbucket_issue = client.last_issue(repo)
+
+ return unless last_bitbucket_issue
+
+ Issue.track_namespace_iid!(project.project_namespace, last_bitbucket_issue.iid)
+ end
+
def build_labels_hash
labels = {}
project.labels.each { |l| labels[l.title.to_s] = l.id }
labels
end
+
+ def repo
+ @repo ||= client.repo(project.import_source)
+ end
end
end
end
diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb
index a18d50e8fce..f7b1753a9f9 100644
--- a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb
+++ b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb
@@ -45,6 +45,8 @@ module Gitlab
merge_request.assignee_ids = [author_id]
merge_request.reviewer_ids = reviewers
merge_request.save!
+
+ metrics.merge_requests_counter.increment
end
log_info(import_stage: 'import_pull_request', message: 'finished', iid: object[:iid])
diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb
index 8ea8b1562f2..934e4ee1720 100644
--- a/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb
+++ b/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb
@@ -4,21 +4,22 @@ module Gitlab
module BitbucketImport
module Importers
class PullRequestNotesImporter
- include Loggable
- include ErrorTracking
+ include ParallelScheduling
def initialize(project, hash)
@project = project
- @importer = Gitlab::BitbucketImport::Importer.new(project)
+ @formatter = Gitlab::ImportFormatter.new
+ @user_finder = UserFinder.new(project)
+ @ref_converter = Gitlab::BitbucketImport::RefConverter.new(project)
@object = hash.with_indifferent_access
+ @position_map = {}
+ @discussion_map = {}
end
def execute
log_info(import_stage: 'import_pull_request_notes', message: 'starting', iid: object[:iid])
- merge_request = project.merge_requests.find_by(iid: object[:iid]) # rubocop: disable CodeReuse/ActiveRecord
-
- importer.import_pull_request_comments(merge_request, merge_request) if merge_request
+ import_pull_request_comments if merge_request
log_info(import_stage: 'import_pull_request_notes', message: 'finished', iid: object[:iid])
rescue StandardError => e
@@ -27,7 +28,116 @@ module Gitlab
private
- attr_reader :object, :project, :importer
+ attr_reader :object, :project, :formatter, :user_finder, :ref_converter, :discussion_map, :position_map
+
+ def import_pull_request_comments
+ inline_comments, pr_comments = comments.partition(&:inline?)
+
+ import_inline_comments(inline_comments)
+ import_standalone_pr_comments(pr_comments)
+ end
+
+ def import_inline_comments(inline_comments)
+ children, parents = inline_comments.partition(&:has_parent?)
+
+ parents.each do |comment|
+ position_map[comment.iid] = build_position(comment)
+
+ import_comment(comment)
+ end
+
+ children.each do |comment|
+ position_map[comment.iid] = position_map.fetch(comment.parent_id, nil)
+
+ import_comment(comment)
+ end
+ end
+
+ def import_comment(comment)
+ position = position_map[comment.iid]
+ discussion_id = discussion_map[comment.parent_id]
+
+ note = create_diff_note(comment, position, discussion_id)
+
+ discussion_map[comment.iid] = note&.discussion_id
+ end
+
+ def create_diff_note(comment, position, discussion_id)
+ attributes = pull_request_comment_attributes(comment)
+ attributes.merge!(position: position, type: 'DiffNote', discussion_id: discussion_id)
+
+ note = merge_request.notes.build(attributes)
+
+ return note if note.save
+
+ # Bitbucket supports the ability to comment on any line, not just the
+ # line in the diff. If we can't add the note as a DiffNote, fallback to creating
+ # a regular note.
+
+ log_info(import_stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid)
+ create_fallback_diff_note(comment, position)
+ rescue StandardError => e
+ Gitlab::ErrorTracking.log_exception(
+ e,
+ import_stage: 'create_diff_note', comment_id: comment.iid, error: e.message
+ )
+
+ nil
+ end
+
+ def create_fallback_diff_note(comment, position)
+ attributes = pull_request_comment_attributes(comment)
+ note = "*Comment on"
+
+ note += " #{position.old_path}:#{position.old_line} -->" if position&.old_line
+ note += " #{position.new_path}:#{position.new_line}" if position&.new_line
+ note += "*\n\n#{comment.note}"
+
+ attributes[:note] = note
+ merge_request.notes.create!(attributes)
+ end
+
+ def build_position(pr_comment)
+ params = {
+ diff_refs: merge_request.diff_refs,
+ old_path: pr_comment.file_path,
+ new_path: pr_comment.file_path,
+ old_line: pr_comment.old_pos,
+ new_line: pr_comment.new_pos
+ }
+
+ Gitlab::Diff::Position.new(params)
+ end
+
+ def import_standalone_pr_comments(pr_comments)
+ pr_comments.each do |comment|
+ attributes = pull_request_comment_attributes(comment)
+ merge_request.notes.create!(attributes)
+ end
+ end
+
+ def pull_request_comment_attributes(comment)
+ {
+ project: project,
+ author_id: user_finder.gitlab_user_id(project, comment.author),
+ note: comment_note(comment),
+ created_at: comment.created_at,
+ updated_at: comment.updated_at
+ }
+ end
+
+ def comment_note(comment)
+ author = formatter.author_line(comment.author) unless user_finder.find_user_id(comment.author)
+ author.to_s + ref_converter.convert_note(comment.note.to_s)
+ end
+
+ def merge_request
+ @merge_request ||= project.merge_requests.iid_in(object[:iid]).first
+ end
+
+ def comments
+ client.pull_request_comments(project.import_source, merge_request.iid).reject(&:deleted?)
+ end
end
end
end
diff --git a/lib/gitlab/bitbucket_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_import/importers/repository_importer.rb
index b8c0ba69d37..9be7ed99436 100644
--- a/lib/gitlab/bitbucket_import/importers/repository_importer.rb
+++ b/lib/gitlab/bitbucket_import/importers/repository_importer.rb
@@ -19,6 +19,7 @@ module Gitlab
validate_repository_size!
+ set_default_branch
update_clone_time
end
@@ -76,6 +77,16 @@ module Gitlab
def validate_repository_size!
# Defined in EE
end
+
+ def set_default_branch
+ default_branch = client.repo(project.import_source).default_branch
+
+ project.change_head(default_branch) if default_branch
+ end
+
+ def client
+ Bitbucket::Client.new(project.import_data.credentials)
+ end
end
end
end
diff --git a/lib/gitlab/bitbucket_import/loggable.rb b/lib/gitlab/bitbucket_import/loggable.rb
index eda3cc96d4d..aeae993b9eb 100644
--- a/lib/gitlab/bitbucket_import/loggable.rb
+++ b/lib/gitlab/bitbucket_import/loggable.rb
@@ -19,6 +19,10 @@ module Gitlab
logger.error(log_data(messages))
end
+ def metrics
+ Gitlab::Import::Metrics.new(:bitbucket_importer, project)
+ end
+
private
def logger
diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb
index 15b38188f13..a359236e150 100644
--- a/lib/gitlab/checks/diff_check.rb
+++ b/lib/gitlab/checks/diff_check.rb
@@ -31,8 +31,7 @@ module Gitlab
def treeish_objects
objects = commits
- return objects unless project.repository.empty? &&
- Feature.enabled?(:verify_push_rules_for_first_commit, project)
+ return objects unless project.repository.empty?
# It's a special case for the push to the empty repository
#
diff --git a/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb b/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb
index 35f969dbb46..b8c6bdee1bb 100644
--- a/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb
+++ b/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb
@@ -6,7 +6,7 @@ module Gitlab
class AnyOversizedBlobs
def initialize(project:, changes:, file_size_limit_megabytes:)
@project = project
- @newrevs = changes.pluck(:newrev).compact # rubocop:disable CodeReuse/ActiveRecord just plucking from an array
+ @newrevs = changes.pluck(:newrev).compact # rubocop:disable CodeReuse/ActiveRecord -- Array#pluck
@file_size_limit_megabytes = file_size_limit_megabytes
end
diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb
index 21fc2980cdc..791b8a963e9 100644
--- a/lib/gitlab/ci/ansi2json/line.rb
+++ b/lib/gitlab/ci/ansi2json/line.rb
@@ -35,13 +35,15 @@ module Gitlab
end
attr_reader :offset, :sections, :segments, :current_segment,
- :section_header, :section_duration, :section_options
+ :section_header, :section_footer, :section_duration,
+ :section_options
def initialize(offset:, style:, sections: [])
@offset = offset
@segments = []
@sections = sections
@section_header = false
+ @section_footer = false
@duration = nil
@current_segment = Segment.new(style: style)
end
@@ -79,6 +81,10 @@ module Gitlab
@section_header = true
end
+ def set_as_section_footer
+ @section_footer = true
+ end
+
def set_section_duration(duration_in_seconds)
normalized_duration_in_seconds = duration_in_seconds.to_i.clamp(0, 1.year)
duration = ActiveSupport::Duration.build(normalized_duration_in_seconds)
@@ -103,6 +109,7 @@ module Gitlab
{ offset: offset, content: @segments }.tap do |result|
result[:section] = sections.last if sections.any?
result[:section_header] = true if @section_header
+ result[:section_footer] = true if @section_footer
result[:section_duration] = @section_duration if @section_duration
result[:section_options] = @section_options if @section_options
end
diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb
index 3aec1cde1bc..6cf76fbbb51 100644
--- a/lib/gitlab/ci/ansi2json/state.rb
+++ b/lib/gitlab/ci/ansi2json/state.rb
@@ -49,6 +49,7 @@ module Gitlab
duration = timestamp.to_i - @open_sections[section].to_i
@current_line.set_section_duration(duration)
+ @current_line.set_as_section_footer
@open_sections.delete(section)
end
diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb
index 48b138b0258..bbcdcd7d389 100644
--- a/lib/gitlab/ci/build/context/build.rb
+++ b/lib/gitlab/ci/build/context/build.rb
@@ -33,13 +33,9 @@ module Gitlab
# Assigning tags and needs is slow and they are not needed for rules
# evaluation since we don't use them to compute the variables at this point.
def build_attributes
- if pipeline.reduced_build_attributes_list_for_rules?
- attributes
- .except(:tag_list, :needs_attributes)
- .merge!(pipeline_attributes, ci_stage_attributes)
- else
- attributes.merge(pipeline_attributes, ci_stage_attributes)
- end
+ attributes
+ .except(:tag_list, :needs_attributes)
+ .merge!(pipeline_attributes, ci_stage_attributes)
end
def ci_stage_attributes
diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb
index df2b2a14fc6..50731d54fc0 100644
--- a/lib/gitlab/ci/components/instance_path.rb
+++ b/lib/gitlab/ci/components/instance_path.rb
@@ -18,7 +18,6 @@ module Gitlab
def initialize(address:)
@full_path, @version = address.to_s.split('@', 2)
@host = Settings.gitlab_ci['component_fqdn']
- @component_project = ::Ci::Catalog::ComponentsProject.new(project, sha)
end
def fetch_content!(current_user:)
@@ -27,7 +26,8 @@ module Gitlab
raise Gitlab::Access::AccessDeniedError unless Ability.allowed?(current_user, :download_code, project)
- @component_project.fetch_component(component_name)
+ component_project = ::Ci::Catalog::ComponentsProject.new(project, sha)
+ component_project.fetch_component(component_name)
end
def project
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index bf8a99ef45e..5fcafcba829 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -14,7 +14,7 @@ module Gitlab
ALLOWED_KEYS = %i[tags script image services start_in artifacts
cache dependencies before_script after_script hooks
coverage retry parallel interruptible timeout
- release id_tokens publish].freeze
+ release id_tokens publish pages].freeze
validations do
validates :config, allowed_keys: Gitlab::Ci::Config::Entry::Job.allowed_keys + PROCESSABLE_ALLOWED_KEYS
@@ -40,13 +40,19 @@ module Gitlab
if needs_value[:job].nil? && needs_value[:cross_dependency].present?
errors.add(:needs, "corresponding to dependencies must be from the same pipeline")
else
- missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord (Array#pluck)
+ missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord -- Array#pluck
errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") if missing_needs.any?
end
end
- validates :publish, absence: { message: "can only be used within a `pages` job" }, unless: -> { pages_job? }
+ validates :publish,
+ absence: { message: "can only be used within a `pages` job" },
+ unless: -> { pages_job? }
+
+ validates :pages,
+ absence: { message: "can only be used within a `pages` job" },
+ unless: -> { pages_job? }
end
entry :before_script, Entry::Commands,
@@ -127,10 +133,14 @@ module Gitlab
description: 'Path to be published with Pages',
inherit: false
+ entry :pages, ::Gitlab::Ci::Config::Entry::Pages,
+ inherit: false,
+ description: 'Pages configuration.'
+
attributes :script, :tags, :when, :dependencies,
:needs, :retry, :parallel, :start_in,
- :interruptible, :timeout,
- :release, :allow_failure, :publish
+ :interruptible, :timeout, :release,
+ :allow_failure, :publish, :pages
def self.matching?(name, config)
!name.to_s.start_with?('.') &&
@@ -170,7 +180,8 @@ module Gitlab
needs: needs_defined? ? needs_value : nil,
scheduling_type: needs_defined? ? :dag : :stage,
id_tokens: id_tokens_value,
- publish: publish
+ publish: publish,
+ pages: pages
).compact
end
diff --git a/lib/gitlab/ci/config/entry/pages.rb b/lib/gitlab/ci/config/entry/pages.rb
new file mode 100644
index 00000000000..57d9e944f51
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/pages.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module Entry
+ ##
+ # Entry that represents the pages path prefix
+ # Entry that represents the pages attributes
+ #
+ class Pages < ::Gitlab::Config::Entry::Node
+ ALLOWED_KEYS = %i[path_prefix].freeze
+
+ include ::Gitlab::Config::Entry::Attributable
+ include ::Gitlab::Config::Entry::Validatable
+
+ attributes ALLOWED_KEYS
+
+ validations do
+ validates :config, type: Hash
+ validates :config, allowed_keys: ALLOWED_KEYS
+
+ with_options allow_nil: true do
+ validates :path_prefix, type: String
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb
index 88734ac1186..d0e9a9afc51 100644
--- a/lib/gitlab/ci/config/entry/processable.rb
+++ b/lib/gitlab/ci/config/entry/processable.rb
@@ -25,6 +25,8 @@ module Gitlab
validates :name, type: Symbol
validates :name, length: { maximum: 255 }
+ validates :config, mutually_exclusive_keys: %i[script trigger]
+
validates :config, disallowed_keys: {
in: %i[only except start_in],
message: 'key may not be used with `rules`',
diff --git a/lib/gitlab/ci/config/header/input.rb b/lib/gitlab/ci/config/header/input.rb
index dcb96006459..08ee70b6290 100644
--- a/lib/gitlab/ci/config/header/input.rb
+++ b/lib/gitlab/ci/config/header/input.rb
@@ -11,17 +11,24 @@ module Gitlab
include ::Gitlab::Config::Entry::Validatable
include ::Gitlab::Config::Entry::Attributable
- ALLOWED_KEYS = %i[default description regex type].freeze
+ ALLOWED_KEYS = %i[default description options regex type].freeze
+ ALLOWED_OPTIONS_LIMIT = 50
attributes ALLOWED_KEYS, prefix: :input
validations do
validates :config, type: Hash, allowed_keys: ALLOWED_KEYS
validates :key, alphanumeric: true
- validates :input_default, alphanumeric: true, allow_nil: true
validates :input_description, alphanumeric: true, allow_nil: true
validates :input_regex, type: String, allow_nil: true
validates :input_type, allow_nil: true, allowed_values: Interpolation::Inputs.input_types
+ validates :input_options, type: Array, allow_nil: true
+
+ validate do
+ if input_options&.size.to_i > ALLOWED_OPTIONS_LIMIT
+ errors.add(:config, "cannot define more than #{ALLOWED_OPTIONS_LIMIT} options")
+ end
+ end
end
end
end
diff --git a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb
index ba519776635..987268b0525 100644
--- a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb
+++ b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb
@@ -20,11 +20,6 @@ module Gitlab
raise NotImplementedError
end
- # Checks whether the provided value is of the given type
- def valid_value?(value)
- raise NotImplementedError
- end
-
attr_reader :errors, :name, :spec, :value
def initialize(name:, spec:, value:)
@@ -54,20 +49,39 @@ module Gitlab
private
def validate!
- return error('required value has not been provided') if required_input? && value.nil?
+ validate_required
+
+ return if errors.present?
- # validate default value
- if !required_input? && !valid_value?(default)
- return error("default value is not a #{self.class.type_name}")
- end
+ run_validations(default, default: true) unless required_input?
- # validate provided value
- return error("provided value is not a #{self.class.type_name}") unless valid_value?(actual_value)
+ run_validations(value) unless value.nil?
+ end
+
+ def validate_required
+ error('required value has not been provided') if required_input? && value.nil?
+ end
- validate_regex!
+ def run_validations(value, default: false)
+ validate_type(value, default)
+ validate_options(value)
+ validate_regex(value, default)
end
- def validate_regex!
+ # Type validations are done separately for different input types.
+ def validate_type(_value, _default)
+ raise NotImplementedError
+ end
+
+ # Options can be either StringInput or NumberInput and are validated accordingly.
+ def validate_options(_value)
+ return unless options
+
+ error('Options can only be used with string and number inputs')
+ end
+
+ # Regex can be only be a StringInput and is validated accordingly.
+ def validate_regex(_value, _default)
return unless spec.key?(:regex)
error('RegEx validation can only be used with string inputs')
@@ -96,6 +110,10 @@ module Gitlab
def default
spec[:default]
end
+
+ def options
+ spec[:options]
+ end
end
end
end
diff --git a/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb b/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb
index 0293c01a5a8..4c34f7e7fdd 100644
--- a/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb
+++ b/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb
@@ -6,6 +6,8 @@ module Gitlab
module Interpolation
class Inputs
class BooleanInput < BaseInput
+ extend ::Gitlab::Utils::Override
+
def self.matches?(spec)
spec.is_a?(Hash) && spec[:type] == type_name
end
@@ -14,8 +16,11 @@ module Gitlab
'boolean'
end
- def valid_value?(value)
- [true, false].include?(value)
+ override :validate_type
+ def validate_type(value, default)
+ return if [true, false].include?(value)
+
+ error("#{default ? 'default' : 'provided'} value is not a boolean")
end
end
end
diff --git a/lib/gitlab/ci/config/interpolation/inputs/number_input.rb b/lib/gitlab/ci/config/interpolation/inputs/number_input.rb
index 314315d2b6d..59bc057749a 100644
--- a/lib/gitlab/ci/config/interpolation/inputs/number_input.rb
+++ b/lib/gitlab/ci/config/interpolation/inputs/number_input.rb
@@ -6,6 +6,8 @@ module Gitlab
module Interpolation
class Inputs
class NumberInput < BaseInput
+ extend ::Gitlab::Utils::Override
+
def self.matches?(spec)
spec.is_a?(Hash) && spec[:type] == type_name
end
@@ -14,8 +16,19 @@ module Gitlab
'number'
end
- def valid_value?(value)
- value.is_a?(Numeric)
+ override :validate_type
+ def validate_type(value, default)
+ return if value.is_a?(Numeric)
+
+ error("#{default ? 'default' : 'provided'} value is not a number")
+ end
+
+ override :validate_options
+ def validate_options(value)
+ return unless options && value
+ return if options.include?(value)
+
+ error("`#{value}` cannot be used because it is not in the list of the allowed options")
end
end
end
diff --git a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb
index 3f40e851f11..01b9d34a883 100644
--- a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb
+++ b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb
@@ -6,6 +6,8 @@ module Gitlab
module Interpolation
class Inputs
class StringInput < BaseInput
+ extend ::Gitlab::Utils::Override
+
def self.matches?(spec)
# The input spec can be `nil` when using a minimal specification
# and also when `type` is not specified.
@@ -22,24 +24,32 @@ module Gitlab
'string'
end
- def valid_value?(value)
- value.nil? || value.is_a?(String)
+ override :validate_type
+ def validate_type(value, default)
+ return if value.is_a?(String)
+
+ error("#{default ? 'default' : 'provided'} value is not a string")
+ end
+
+ override :validate_options
+ def validate_options(value)
+ return unless options && value
+ return if options.include?(value)
+
+ error("`#{value}` cannot be used because it is not in the list of allowed options")
end
private
- def validate_regex!
+ override :validate_regex
+ def validate_regex(value, default)
return unless spec.key?(:regex)
safe_regex = ::Gitlab::UntrustedRegexp.new(spec[:regex])
- return if safe_regex.match?(actual_value)
+ return if safe_regex.match?(value)
- if value.nil?
- error('default value does not match required RegEx pattern')
- else
- error('provided value does not match required RegEx pattern')
- end
+ error("#{default ? 'default' : 'provided'} value does not match required RegEx pattern")
rescue RegexpError
error('invalid regular expression')
end
diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb
index 29beba4774a..90db9d13d85 100644
--- a/lib/gitlab/ci/jwt_v2.rb
+++ b/lib/gitlab/ci/jwt_v2.rb
@@ -25,13 +25,26 @@ module Gitlab
def reserved_claims
super.merge({
- iss: Settings.gitlab.base_url,
+ iss: Feature.enabled?(:oidc_issuer_url) ? Gitlab.config.gitlab.url : Settings.gitlab.base_url,
sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}",
- aud: aud,
- user_identities: user_identities
+ aud: aud
}.compact)
end
+ def custom_claims
+ additional_custom_claims = {
+ runner_id: runner&.id,
+ runner_environment: runner_environment,
+ sha: pipeline.sha,
+ project_visibility: Gitlab::VisibilityLevel.string_level(project.visibility_level),
+ user_identities: user_identities
+ }.compact
+
+ mapper = ClaimMapper.new(project_config, pipeline)
+
+ super.merge(additional_custom_claims).merge(mapper.to_h)
+ end
+
def user_identities
return unless user&.pass_user_identities_to_ci_jwt
@@ -43,17 +56,6 @@ module Gitlab
end
end
- def custom_claims
- mapper = ClaimMapper.new(project_config, pipeline)
-
- super.merge({
- runner_id: runner&.id,
- runner_environment: runner_environment,
- sha: pipeline.sha,
- project_visibility: Gitlab::VisibilityLevel.string_level(project.visibility_level)
- }).merge(mapper.to_h)
- end
-
def project_config
Gitlab::Ci::ProjectConfig.new(
project: project,
diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb
index 3dc73544208..35548358c57 100644
--- a/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb
+++ b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb
@@ -15,7 +15,8 @@ module Gitlab
SUPPORTED_SCHEMA_VERSION = '1'
GITLAB_PREFIX = 'gitlab:'
SOURCE_PARSERS = {
- 'dependency_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning
+ 'dependency_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning,
+ 'container_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::ContainerScanning
}.freeze
SUPPORTED_PROPERTIES = %w[
meta:schema_version
@@ -24,6 +25,10 @@ module Gitlab
dependency_scanning:source_file:path
dependency_scanning:package_manager:name
dependency_scanning:language:name
+ container_scanning:image:name
+ container_scanning:image:tag
+ container_scanning:operating_system:name
+ container_scanning:operating_system:version
].freeze
def self.parse_source(...)
diff --git a/lib/gitlab/ci/parsers/sbom/source/base_source.rb b/lib/gitlab/ci/parsers/sbom/source/base_source.rb
new file mode 100644
index 00000000000..744555aa25a
--- /dev/null
+++ b/lib/gitlab/ci/parsers/sbom/source/base_source.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Parsers
+ module Sbom
+ module Source
+ class BaseSource
+ REQUIRED_ATTRIBUTES = [].freeze
+
+ def self.source(...)
+ new(...).source
+ end
+
+ def initialize(data)
+ @data = data
+ end
+
+ def source
+ return unless required_attributes_present?
+
+ ::Gitlab::Ci::Reports::Sbom::Source.new(
+ type: type,
+ data: data
+ )
+ end
+
+ private
+
+ attr_reader :data
+
+ # Implement in child class
+ # returns a symbol of the source type
+ def type; end
+
+ def required_attributes_present?
+ self.class::REQUIRED_ATTRIBUTES.all? do |keys|
+ data.dig(*keys).present?
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/parsers/sbom/source/container_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/container_scanning.rb
new file mode 100644
index 00000000000..33f9631c424
--- /dev/null
+++ b/lib/gitlab/ci/parsers/sbom/source/container_scanning.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Parsers
+ module Sbom
+ module Source
+ class ContainerScanning < BaseSource
+ REQUIRED_ATTRIBUTES = [
+ %w[image name],
+ %w[image tag]
+ ].freeze
+
+ OPERATING_SYSTEM_ATTRIBUTES = [
+ %w[operating_system name],
+ %w[operating_system version]
+ ].freeze
+
+ private
+
+ def type
+ :container_scanning
+ end
+
+ def required_attributes_present?
+ operating_system_attributes_valid? && super
+ end
+
+ def operating_system_attributes_valid?
+ return true if data['operating_system'].blank?
+
+ OPERATING_SYSTEM_ATTRIBUTES.all? do |keys|
+ data.dig(*keys).present?
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb
index c76a4309779..fc5a7606e39 100644
--- a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb
+++ b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb
@@ -5,36 +5,15 @@ module Gitlab
module Parsers
module Sbom
module Source
- class DependencyScanning
+ class DependencyScanning < BaseSource
REQUIRED_ATTRIBUTES = [
%w[input_file path]
].freeze
- def self.source(...)
- new(...).source
- end
-
- def initialize(data)
- @data = data
- end
-
- def source
- return unless required_attributes_present?
-
- ::Gitlab::Ci::Reports::Sbom::Source.new(
- type: :dependency_scanning,
- data: data
- )
- end
-
private
- attr_reader :data
-
- def required_attributes_present?
- REQUIRED_ATTRIBUTES.all? do |keys|
- data.dig(*keys).present?
- end
+ def type
+ :dependency_scanning
end
end
end
diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb
index 9032faa66d4..be6c6c2558b 100644
--- a/lib/gitlab/ci/parsers/security/common.rb
+++ b/lib/gitlab/ci/parsers/security/common.rb
@@ -141,7 +141,7 @@ module Gitlab
project_id: @project.id,
found_by_pipeline: report.pipeline,
vulnerability_finding_signatures_enabled: @signatures_enabled,
- cvss: data['cvss'] || []
+ cvss: data['cvss_vectors'] || []
)
)
end
diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
index e39482481c7..e2a8044b708 100644
--- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
+++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
@@ -7,14 +7,14 @@ module Gitlab
module Validators
class SchemaValidator
SUPPORTED_VERSIONS = {
- cluster_image_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6],
- container_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6],
- coverage_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6],
- dast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6],
- api_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6],
- dependency_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6],
- sast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6],
- secret_detection: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6]
+ cluster_image_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7],
+ container_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7],
+ coverage_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7],
+ dast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7],
+ api_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7],
+ dependency_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7],
+ sast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7],
+ secret_detection: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7]
}.freeze
VERSIONS_TO_REMOVE_IN_17_0 = %w[].freeze
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/cluster-image-scanning-report-format.json
new file mode 100644
index 00000000000..e27096d071f
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/cluster-image-scanning-report-format.json
@@ -0,0 +1,1085 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json",
+ "title": "Report format for GitLab Cluster Image Scanning",
+ "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).",
+ "definitions": {
+ "detail_type": {
+ "oneOf": [
+ {
+ "$ref": "#/definitions/named_list"
+ },
+ {
+ "$ref": "#/definitions/list"
+ },
+ {
+ "$ref": "#/definitions/table"
+ },
+ {
+ "$ref": "#/definitions/text"
+ },
+ {
+ "$ref": "#/definitions/url"
+ },
+ {
+ "$ref": "#/definitions/code"
+ },
+ {
+ "$ref": "#/definitions/value"
+ },
+ {
+ "$ref": "#/definitions/diff"
+ },
+ {
+ "$ref": "#/definitions/markdown"
+ },
+ {
+ "$ref": "#/definitions/commit"
+ },
+ {
+ "$ref": "#/definitions/file_location"
+ },
+ {
+ "$ref": "#/definitions/module_location"
+ }
+ ]
+ },
+ "text_value": {
+ "type": "string"
+ },
+ "named_field": {
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "$ref": "#/definitions/text_value",
+ "type": "string",
+ "minLength": 1
+ },
+ "description": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "named_list": {
+ "type": "object",
+ "description": "An object with named and typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "named-list"
+ },
+ "items": {
+ "type": "object",
+ "patternProperties": {
+ "^.*$": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/named_field"
+ },
+ {
+ "$ref": "#/definitions/detail_type"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "list": {
+ "type": "object",
+ "description": "A list of typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "list"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ },
+ "table": {
+ "type": "object",
+ "description": "A table of typed fields",
+ "required": [
+ "type",
+ "rows"
+ ],
+ "properties": {
+ "type": {
+ "const": "table"
+ },
+ "header": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ },
+ "rows": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ }
+ },
+ "text": {
+ "type": "object",
+ "description": "Raw text",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "text"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "url": {
+ "type": "object",
+ "description": "A single URL",
+ "required": [
+ "type",
+ "href"
+ ],
+ "properties": {
+ "type": {
+ "const": "url"
+ },
+ "text": {
+ "$ref": "#/definitions/text_value"
+ },
+ "href": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "http://mysite.com"
+ ]
+ }
+ }
+ },
+ "code": {
+ "type": "object",
+ "description": "A codeblock",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "code"
+ },
+ "value": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string",
+ "description": "A programming language"
+ }
+ }
+ },
+ "value": {
+ "type": "object",
+ "description": "A field that can store a range of types of value",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "value"
+ },
+ "value": {
+ "type": [
+ "number",
+ "string",
+ "boolean"
+ ]
+ }
+ }
+ },
+ "diff": {
+ "type": "object",
+ "description": "A diff",
+ "required": [
+ "type",
+ "before",
+ "after"
+ ],
+ "properties": {
+ "type": {
+ "const": "diff"
+ },
+ "before": {
+ "type": "string"
+ },
+ "after": {
+ "type": "string"
+ }
+ }
+ },
+ "markdown": {
+ "type": "object",
+ "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "markdown"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value",
+ "examples": [
+ "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+ ]
+ }
+ }
+ },
+ "commit": {
+ "type": "object",
+ "description": "A commit/tag/branch within the GitLab project",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "commit"
+ },
+ "value": {
+ "type": "string",
+ "description": "The commit SHA",
+ "minLength": 1
+ }
+ }
+ },
+ "file_location": {
+ "type": "object",
+ "description": "A location within a file in the project",
+ "required": [
+ "type",
+ "file_name",
+ "line_start"
+ ],
+ "properties": {
+ "type": {
+ "const": "file-location"
+ },
+ "file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "line_start": {
+ "type": "integer"
+ },
+ "line_end": {
+ "type": "integer"
+ }
+ }
+ },
+ "module_location": {
+ "type": "object",
+ "description": "A location within a binary module of the form module+relative_offset",
+ "required": [
+ "type",
+ "module_name",
+ "offset"
+ ],
+ "properties": {
+ "type": {
+ "const": "module-location"
+ },
+ "module_name": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "compiled_binary"
+ ]
+ },
+ "offset": {
+ "type": "integer",
+ "examples": [
+ 100
+ ]
+ }
+ }
+ }
+ },
+ "self": {
+ "version": "15.0.7"
+ },
+ "type": "object",
+ "required": [
+ "scan",
+ "version",
+ "vulnerabilities"
+ ],
+ "additionalProperties": true,
+ "properties": {
+ "scan": {
+ "type": "object",
+ "required": [
+ "analyzer",
+ "end_time",
+ "scanner",
+ "start_time",
+ "status",
+ "type"
+ ],
+ "properties": {
+ "end_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-01-28T03:26:02"
+ ]
+ },
+ "messages": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Communication intended for the initiator of a scan.",
+ "required": [
+ "level",
+ "value"
+ ],
+ "properties": {
+ "level": {
+ "type": "string",
+ "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+ "enum": [
+ "info",
+ "warn",
+ "fatal"
+ ],
+ "examples": [
+ "info"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "The message to communicate.",
+ "minLength": 1,
+ "examples": [
+ "Permission denied, scanning aborted"
+ ]
+ }
+ }
+ }
+ },
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
+ "analyzer": {
+ "type": "object",
+ "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "gitlab-dast"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the analyzer, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "GitLab DAST"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "pattern": "^https?://.+",
+ "description": "A link to more information about the analyzer.",
+ "examples": [
+ "https://docs.gitlab.com/ee/user/application_security/dast"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the analyzer.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ }
+ }
+ },
+ "scanner": {
+ "type": "object",
+ "description": "Object defining the scanner used to perform the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the scanner.",
+ "minLength": 1,
+ "examples": [
+ "my-sast-scanner"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the scanner, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "My SAST Scanner"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "A link to more information about the scanner.",
+ "examples": [
+ "https://scanner.url"
+ ]
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the scanner.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the scanner.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ }
+ }
+ },
+ "start_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-02-14T16:01:59"
+ ]
+ },
+ "status": {
+ "type": "string",
+ "description": "Result of the scan.",
+ "enum": [
+ "success",
+ "failure"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "description": "Type of the scan.",
+ "enum": [
+ "cluster_image_scanning"
+ ]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ }
+ }
+ },
+ "schema": {
+ "type": "string",
+ "description": "URI pointing to the validating security report schema.",
+ "pattern": "^https?://.+"
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the schema to which the JSON report conforms.",
+ "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+ },
+ "vulnerabilities": {
+ "type": "array",
+ "description": "Array of vulnerability objects.",
+ "items": {
+ "type": "object",
+ "description": "Describes the vulnerability using GitLab Flavored Markdown",
+ "required": [
+ "id",
+ "identifiers",
+ "location"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "maxLength": 255,
+ "description": "The name of the vulnerability. This must not include the finding's specific information."
+ },
+ "description": {
+ "type": "string",
+ "maxLength": 1048576,
+ "description": "A long text section describing the vulnerability more fully."
+ },
+ "severity": {
+ "type": "string",
+ "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+ "enum": [
+ "Info",
+ "Unknown",
+ "Low",
+ "Medium",
+ "High",
+ "Critical"
+ ]
+ },
+ "solution": {
+ "type": "string",
+ "maxLength": 7000,
+ "description": "Explanation of how to fix the vulnerability."
+ },
+ "identifiers": {
+ "type": "array",
+ "minItems": 1,
+ "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ },
+ "cvss_vectors": {
+ "type": "array",
+ "minItems": 1,
+ "maxItems": 10,
+ "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.",
+ "items": {
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 16,
+ "maxLength": 128,
+ "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ },
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 32,
+ "maxLength": 128,
+ "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ }
+ ]
+ }
+ },
+ "links": {
+ "type": "array",
+ "description": "An array of references to external documentation or articles that describe the vulnerability.",
+ "items": {
+ "type": "object",
+ "required": [
+ "url"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the vulnerability details link."
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the vulnerability details document.",
+ "pattern": "^(https?|ftp)://.+"
+ }
+ }
+ }
+ },
+ "details": {
+ "$ref": "#/definitions/named_list/properties/items"
+ },
+ "tracking": {
+ "type": "object",
+ "description": "Describes how this vulnerability should be tracked as the project changes.",
+ "oneOf": [
+ {
+ "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
+ "required": [
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "source"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "description": "An item that should be tracked using source-specific tracking methods.",
+ "type": "object",
+ "required": [
+ "signatures"
+ ],
+ "properties": {
+ "file": {
+ "type": "string",
+ "description": "Path to the file where the vulnerability is located."
+ },
+ "start_line": {
+ "type": "number",
+ "description": "The first line of the file that includes the vulnerability."
+ },
+ "end_line": {
+ "type": "number",
+ "description": "The last line of the file that includes the vulnerability."
+ },
+ "signatures": {
+ "type": "array",
+ "description": "An array of calculated tracking signatures for this tracking item.",
+ "minItems": 1,
+ "items": {
+ "description": "A calculated tracking signature value and metadata.",
+ "type": "object",
+ "required": [
+ "algorithm",
+ "value"
+ ],
+ "properties": {
+ "algorithm": {
+ "type": "string",
+ "description": "The algorithm used to generate the signature."
+ },
+ "value": {
+ "type": "string",
+ "description": "The result of this signature algorithm."
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "Each tracking type must declare its own type."
+ }
+ }
+ },
+ "flags": {
+ "description": "Flags that can be attached to vulnerabilities.",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Informational flags identified and assigned to a vulnerability.",
+ "required": [
+ "type",
+ "origin",
+ "description"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Result of the scan.",
+ "enum": [
+ "flagged-as-likely-false-positive"
+ ]
+ },
+ "origin": {
+ "minLength": 1,
+ "description": "Tool that issued the flag.",
+ "type": "string"
+ },
+ "description": {
+ "minLength": 1,
+ "description": "What the flag is about.",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "location": {
+ "type": "object",
+ "description": "Identifies the vulnerability's location.",
+ "required": [
+ "dependency",
+ "image",
+ "kubernetes_resource"
+ ],
+ "properties": {
+ "dependency": {
+ "type": "object",
+ "description": "Describes the dependency of a project where the vulnerability is located.",
+ "required": [
+ "package",
+ "version"
+ ],
+ "properties": {
+ "package": {
+ "type": "object",
+ "description": "Provides information on the package where the vulnerability is located.",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the package where the vulnerability is located."
+ }
+ }
+ },
+ "version": {
+ "type": "string",
+ "description": "Version of the vulnerable package."
+ },
+ "iid": {
+ "description": "ID that identifies the dependency in the scope of a dependency file.",
+ "type": "number"
+ },
+ "direct": {
+ "type": "boolean",
+ "description": "Tells whether this is a direct, top-level dependency of the scanned project."
+ },
+ "dependency_path": {
+ "type": "array",
+ "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
+ "items": {
+ "type": "object",
+ "required": [
+ "iid"
+ ],
+ "properties": {
+ "iid": {
+ "type": "number",
+ "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
+ }
+ }
+ }
+ }
+ }
+ },
+ "operating_system": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 255,
+ "description": "The operating system that contains the vulnerable package."
+ },
+ "image": {
+ "type": "string",
+ "minLength": 1,
+ "description": "The analyzed Docker image.",
+ "examples": [
+ "index.docker.io/library/nginx:1.21"
+ ]
+ },
+ "kubernetes_resource": {
+ "type": "object",
+ "description": "The specific Kubernetes resource that was scanned.",
+ "required": [
+ "namespace",
+ "kind",
+ "name",
+ "container_name"
+ ],
+ "properties": {
+ "namespace": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 255,
+ "description": "The Kubernetes namespace the resource that had its image scanned.",
+ "examples": [
+ "default",
+ "staging",
+ "production"
+ ]
+ },
+ "kind": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 255,
+ "description": "The Kubernetes kind the resource that had its image scanned.",
+ "examples": [
+ "Deployment",
+ "DaemonSet"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 255,
+ "description": "The name of the resource that had its image scanned.",
+ "examples": [
+ "nginx-ingress"
+ ]
+ },
+ "container_name": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 255,
+ "description": "The name of the container that had its image scanned.",
+ "examples": [
+ "nginx"
+ ]
+ },
+ "agent_id": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 255,
+ "description": "The GitLab ID of the Kubernetes Agent which performed the scan.",
+ "examples": [
+ "1234"
+ ]
+ },
+ "cluster_id": {
+ "type": "string",
+ "minLength": 1,
+ "maxLength": 255,
+ "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.",
+ "examples": [
+ "1234"
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "remediations": {
+ "type": "array",
+ "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+ "items": {
+ "type": "object",
+ "required": [
+ "fixes",
+ "summary",
+ "diff"
+ ],
+ "properties": {
+ "fixes": {
+ "type": "array",
+ "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+ "items": {
+ "type": "object",
+ "required": [
+ "id"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ }
+ }
+ }
+ },
+ "summary": {
+ "type": "string",
+ "minLength": 1,
+ "description": "An overview of how the vulnerabilities were fixed."
+ },
+ "diff": {
+ "type": "string",
+ "minLength": 1,
+ "description": "A base64-encoded remediation code diff, compatible with git apply."
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/container-scanning-report-format.json
new file mode 100644
index 00000000000..94c3b3fc919
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/container-scanning-report-format.json
@@ -0,0 +1,1017 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json",
+ "title": "Report format for GitLab Container Scanning",
+ "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).",
+ "definitions": {
+ "detail_type": {
+ "oneOf": [
+ {
+ "$ref": "#/definitions/named_list"
+ },
+ {
+ "$ref": "#/definitions/list"
+ },
+ {
+ "$ref": "#/definitions/table"
+ },
+ {
+ "$ref": "#/definitions/text"
+ },
+ {
+ "$ref": "#/definitions/url"
+ },
+ {
+ "$ref": "#/definitions/code"
+ },
+ {
+ "$ref": "#/definitions/value"
+ },
+ {
+ "$ref": "#/definitions/diff"
+ },
+ {
+ "$ref": "#/definitions/markdown"
+ },
+ {
+ "$ref": "#/definitions/commit"
+ },
+ {
+ "$ref": "#/definitions/file_location"
+ },
+ {
+ "$ref": "#/definitions/module_location"
+ }
+ ]
+ },
+ "text_value": {
+ "type": "string"
+ },
+ "named_field": {
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "$ref": "#/definitions/text_value",
+ "type": "string",
+ "minLength": 1
+ },
+ "description": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "named_list": {
+ "type": "object",
+ "description": "An object with named and typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "named-list"
+ },
+ "items": {
+ "type": "object",
+ "patternProperties": {
+ "^.*$": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/named_field"
+ },
+ {
+ "$ref": "#/definitions/detail_type"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "list": {
+ "type": "object",
+ "description": "A list of typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "list"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ },
+ "table": {
+ "type": "object",
+ "description": "A table of typed fields",
+ "required": [
+ "type",
+ "rows"
+ ],
+ "properties": {
+ "type": {
+ "const": "table"
+ },
+ "header": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ },
+ "rows": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ }
+ },
+ "text": {
+ "type": "object",
+ "description": "Raw text",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "text"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "url": {
+ "type": "object",
+ "description": "A single URL",
+ "required": [
+ "type",
+ "href"
+ ],
+ "properties": {
+ "type": {
+ "const": "url"
+ },
+ "text": {
+ "$ref": "#/definitions/text_value"
+ },
+ "href": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "http://mysite.com"
+ ]
+ }
+ }
+ },
+ "code": {
+ "type": "object",
+ "description": "A codeblock",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "code"
+ },
+ "value": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string",
+ "description": "A programming language"
+ }
+ }
+ },
+ "value": {
+ "type": "object",
+ "description": "A field that can store a range of types of value",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "value"
+ },
+ "value": {
+ "type": [
+ "number",
+ "string",
+ "boolean"
+ ]
+ }
+ }
+ },
+ "diff": {
+ "type": "object",
+ "description": "A diff",
+ "required": [
+ "type",
+ "before",
+ "after"
+ ],
+ "properties": {
+ "type": {
+ "const": "diff"
+ },
+ "before": {
+ "type": "string"
+ },
+ "after": {
+ "type": "string"
+ }
+ }
+ },
+ "markdown": {
+ "type": "object",
+ "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "markdown"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value",
+ "examples": [
+ "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+ ]
+ }
+ }
+ },
+ "commit": {
+ "type": "object",
+ "description": "A commit/tag/branch within the GitLab project",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "commit"
+ },
+ "value": {
+ "type": "string",
+ "description": "The commit SHA",
+ "minLength": 1
+ }
+ }
+ },
+ "file_location": {
+ "type": "object",
+ "description": "A location within a file in the project",
+ "required": [
+ "type",
+ "file_name",
+ "line_start"
+ ],
+ "properties": {
+ "type": {
+ "const": "file-location"
+ },
+ "file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "line_start": {
+ "type": "integer"
+ },
+ "line_end": {
+ "type": "integer"
+ }
+ }
+ },
+ "module_location": {
+ "type": "object",
+ "description": "A location within a binary module of the form module+relative_offset",
+ "required": [
+ "type",
+ "module_name",
+ "offset"
+ ],
+ "properties": {
+ "type": {
+ "const": "module-location"
+ },
+ "module_name": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "compiled_binary"
+ ]
+ },
+ "offset": {
+ "type": "integer",
+ "examples": [
+ 100
+ ]
+ }
+ }
+ }
+ },
+ "self": {
+ "version": "15.0.7"
+ },
+ "type": "object",
+ "required": [
+ "scan",
+ "version",
+ "vulnerabilities"
+ ],
+ "additionalProperties": true,
+ "properties": {
+ "scan": {
+ "type": "object",
+ "required": [
+ "analyzer",
+ "end_time",
+ "scanner",
+ "start_time",
+ "status",
+ "type"
+ ],
+ "properties": {
+ "end_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-01-28T03:26:02"
+ ]
+ },
+ "messages": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Communication intended for the initiator of a scan.",
+ "required": [
+ "level",
+ "value"
+ ],
+ "properties": {
+ "level": {
+ "type": "string",
+ "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+ "enum": [
+ "info",
+ "warn",
+ "fatal"
+ ],
+ "examples": [
+ "info"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "The message to communicate.",
+ "minLength": 1,
+ "examples": [
+ "Permission denied, scanning aborted"
+ ]
+ }
+ }
+ }
+ },
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
+ "analyzer": {
+ "type": "object",
+ "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "gitlab-dast"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the analyzer, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "GitLab DAST"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "pattern": "^https?://.+",
+ "description": "A link to more information about the analyzer.",
+ "examples": [
+ "https://docs.gitlab.com/ee/user/application_security/dast"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the analyzer.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ }
+ }
+ },
+ "scanner": {
+ "type": "object",
+ "description": "Object defining the scanner used to perform the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the scanner.",
+ "minLength": 1,
+ "examples": [
+ "my-sast-scanner"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the scanner, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "My SAST Scanner"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "A link to more information about the scanner.",
+ "examples": [
+ "https://scanner.url"
+ ]
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the scanner.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the scanner.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ }
+ }
+ },
+ "start_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-02-14T16:01:59"
+ ]
+ },
+ "status": {
+ "type": "string",
+ "description": "Result of the scan.",
+ "enum": [
+ "success",
+ "failure"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "description": "Type of the scan.",
+ "enum": [
+ "container_scanning"
+ ]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ }
+ }
+ },
+ "schema": {
+ "type": "string",
+ "description": "URI pointing to the validating security report schema.",
+ "pattern": "^https?://.+"
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the schema to which the JSON report conforms.",
+ "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+ },
+ "vulnerabilities": {
+ "type": "array",
+ "description": "Array of vulnerability objects.",
+ "items": {
+ "type": "object",
+ "description": "Describes the vulnerability using GitLab Flavored Markdown",
+ "required": [
+ "id",
+ "identifiers",
+ "location"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "maxLength": 255,
+ "description": "The name of the vulnerability. This must not include the finding's specific information."
+ },
+ "description": {
+ "type": "string",
+ "maxLength": 1048576,
+ "description": "A long text section describing the vulnerability more fully."
+ },
+ "severity": {
+ "type": "string",
+ "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+ "enum": [
+ "Info",
+ "Unknown",
+ "Low",
+ "Medium",
+ "High",
+ "Critical"
+ ]
+ },
+ "solution": {
+ "type": "string",
+ "maxLength": 7000,
+ "description": "Explanation of how to fix the vulnerability."
+ },
+ "identifiers": {
+ "type": "array",
+ "minItems": 1,
+ "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ },
+ "cvss_vectors": {
+ "type": "array",
+ "minItems": 1,
+ "maxItems": 10,
+ "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.",
+ "items": {
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 16,
+ "maxLength": 128,
+ "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ },
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 32,
+ "maxLength": 128,
+ "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ }
+ ]
+ }
+ },
+ "links": {
+ "type": "array",
+ "description": "An array of references to external documentation or articles that describe the vulnerability.",
+ "items": {
+ "type": "object",
+ "required": [
+ "url"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the vulnerability details link."
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the vulnerability details document.",
+ "pattern": "^(https?|ftp)://.+"
+ }
+ }
+ }
+ },
+ "details": {
+ "$ref": "#/definitions/named_list/properties/items"
+ },
+ "tracking": {
+ "type": "object",
+ "description": "Describes how this vulnerability should be tracked as the project changes.",
+ "oneOf": [
+ {
+ "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
+ "required": [
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "source"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "description": "An item that should be tracked using source-specific tracking methods.",
+ "type": "object",
+ "required": [
+ "signatures"
+ ],
+ "properties": {
+ "file": {
+ "type": "string",
+ "description": "Path to the file where the vulnerability is located."
+ },
+ "start_line": {
+ "type": "number",
+ "description": "The first line of the file that includes the vulnerability."
+ },
+ "end_line": {
+ "type": "number",
+ "description": "The last line of the file that includes the vulnerability."
+ },
+ "signatures": {
+ "type": "array",
+ "description": "An array of calculated tracking signatures for this tracking item.",
+ "minItems": 1,
+ "items": {
+ "description": "A calculated tracking signature value and metadata.",
+ "type": "object",
+ "required": [
+ "algorithm",
+ "value"
+ ],
+ "properties": {
+ "algorithm": {
+ "type": "string",
+ "description": "The algorithm used to generate the signature."
+ },
+ "value": {
+ "type": "string",
+ "description": "The result of this signature algorithm."
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "Each tracking type must declare its own type."
+ }
+ }
+ },
+ "flags": {
+ "description": "Flags that can be attached to vulnerabilities.",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Informational flags identified and assigned to a vulnerability.",
+ "required": [
+ "type",
+ "origin",
+ "description"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Result of the scan.",
+ "enum": [
+ "flagged-as-likely-false-positive"
+ ]
+ },
+ "origin": {
+ "minLength": 1,
+ "description": "Tool that issued the flag.",
+ "type": "string"
+ },
+ "description": {
+ "minLength": 1,
+ "description": "What the flag is about.",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "location": {
+ "type": "object",
+ "description": "Identifies the vulnerability's location.",
+ "required": [
+ "dependency",
+ "operating_system",
+ "image"
+ ],
+ "properties": {
+ "dependency": {
+ "type": "object",
+ "description": "Describes the dependency of a project where the vulnerability is located.",
+ "required": [
+ "package",
+ "version"
+ ],
+ "properties": {
+ "package": {
+ "type": "object",
+ "description": "Provides information on the package where the vulnerability is located.",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the package where the vulnerability is located."
+ }
+ }
+ },
+ "version": {
+ "type": "string",
+ "description": "Version of the vulnerable package."
+ },
+ "iid": {
+ "description": "ID that identifies the dependency in the scope of a dependency file.",
+ "type": "number"
+ },
+ "direct": {
+ "type": "boolean",
+ "description": "Tells whether this is a direct, top-level dependency of the scanned project."
+ },
+ "dependency_path": {
+ "type": "array",
+ "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
+ "items": {
+ "type": "object",
+ "required": [
+ "iid"
+ ],
+ "properties": {
+ "iid": {
+ "type": "number",
+ "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
+ }
+ }
+ }
+ }
+ }
+ },
+ "operating_system": {
+ "type": "string",
+ "minLength": 1,
+ "description": "The operating system that contains the vulnerable package."
+ },
+ "image": {
+ "type": "string",
+ "minLength": 1,
+ "description": "The analyzed Docker image."
+ },
+ "default_branch_image": {
+ "type": "string",
+ "maxLength": 255,
+ "description": "The name of the image on the default branch."
+ }
+ }
+ }
+ }
+ }
+ },
+ "remediations": {
+ "type": "array",
+ "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+ "items": {
+ "type": "object",
+ "required": [
+ "fixes",
+ "summary",
+ "diff"
+ ],
+ "properties": {
+ "fixes": {
+ "type": "array",
+ "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+ "items": {
+ "type": "object",
+ "required": [
+ "id"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ }
+ }
+ }
+ },
+ "summary": {
+ "type": "string",
+ "minLength": 1,
+ "description": "An overview of how the vulnerabilities were fixed."
+ },
+ "diff": {
+ "type": "string",
+ "minLength": 1,
+ "description": "A base64-encoded remediation code diff, compatible with git apply."
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/coverage-fuzzing-report-format.json
new file mode 100644
index 00000000000..e15fbc3ed56
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/coverage-fuzzing-report-format.json
@@ -0,0 +1,975 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json",
+ "title": "Report format for GitLab Fuzz Testing",
+ "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).",
+ "definitions": {
+ "detail_type": {
+ "oneOf": [
+ {
+ "$ref": "#/definitions/named_list"
+ },
+ {
+ "$ref": "#/definitions/list"
+ },
+ {
+ "$ref": "#/definitions/table"
+ },
+ {
+ "$ref": "#/definitions/text"
+ },
+ {
+ "$ref": "#/definitions/url"
+ },
+ {
+ "$ref": "#/definitions/code"
+ },
+ {
+ "$ref": "#/definitions/value"
+ },
+ {
+ "$ref": "#/definitions/diff"
+ },
+ {
+ "$ref": "#/definitions/markdown"
+ },
+ {
+ "$ref": "#/definitions/commit"
+ },
+ {
+ "$ref": "#/definitions/file_location"
+ },
+ {
+ "$ref": "#/definitions/module_location"
+ }
+ ]
+ },
+ "text_value": {
+ "type": "string"
+ },
+ "named_field": {
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "$ref": "#/definitions/text_value",
+ "type": "string",
+ "minLength": 1
+ },
+ "description": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "named_list": {
+ "type": "object",
+ "description": "An object with named and typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "named-list"
+ },
+ "items": {
+ "type": "object",
+ "patternProperties": {
+ "^.*$": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/named_field"
+ },
+ {
+ "$ref": "#/definitions/detail_type"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "list": {
+ "type": "object",
+ "description": "A list of typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "list"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ },
+ "table": {
+ "type": "object",
+ "description": "A table of typed fields",
+ "required": [
+ "type",
+ "rows"
+ ],
+ "properties": {
+ "type": {
+ "const": "table"
+ },
+ "header": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ },
+ "rows": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ }
+ },
+ "text": {
+ "type": "object",
+ "description": "Raw text",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "text"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "url": {
+ "type": "object",
+ "description": "A single URL",
+ "required": [
+ "type",
+ "href"
+ ],
+ "properties": {
+ "type": {
+ "const": "url"
+ },
+ "text": {
+ "$ref": "#/definitions/text_value"
+ },
+ "href": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "http://mysite.com"
+ ]
+ }
+ }
+ },
+ "code": {
+ "type": "object",
+ "description": "A codeblock",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "code"
+ },
+ "value": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string",
+ "description": "A programming language"
+ }
+ }
+ },
+ "value": {
+ "type": "object",
+ "description": "A field that can store a range of types of value",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "value"
+ },
+ "value": {
+ "type": [
+ "number",
+ "string",
+ "boolean"
+ ]
+ }
+ }
+ },
+ "diff": {
+ "type": "object",
+ "description": "A diff",
+ "required": [
+ "type",
+ "before",
+ "after"
+ ],
+ "properties": {
+ "type": {
+ "const": "diff"
+ },
+ "before": {
+ "type": "string"
+ },
+ "after": {
+ "type": "string"
+ }
+ }
+ },
+ "markdown": {
+ "type": "object",
+ "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "markdown"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value",
+ "examples": [
+ "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+ ]
+ }
+ }
+ },
+ "commit": {
+ "type": "object",
+ "description": "A commit/tag/branch within the GitLab project",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "commit"
+ },
+ "value": {
+ "type": "string",
+ "description": "The commit SHA",
+ "minLength": 1
+ }
+ }
+ },
+ "file_location": {
+ "type": "object",
+ "description": "A location within a file in the project",
+ "required": [
+ "type",
+ "file_name",
+ "line_start"
+ ],
+ "properties": {
+ "type": {
+ "const": "file-location"
+ },
+ "file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "line_start": {
+ "type": "integer"
+ },
+ "line_end": {
+ "type": "integer"
+ }
+ }
+ },
+ "module_location": {
+ "type": "object",
+ "description": "A location within a binary module of the form module+relative_offset",
+ "required": [
+ "type",
+ "module_name",
+ "offset"
+ ],
+ "properties": {
+ "type": {
+ "const": "module-location"
+ },
+ "module_name": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "compiled_binary"
+ ]
+ },
+ "offset": {
+ "type": "integer",
+ "examples": [
+ 100
+ ]
+ }
+ }
+ }
+ },
+ "self": {
+ "version": "15.0.7"
+ },
+ "type": "object",
+ "required": [
+ "scan",
+ "version",
+ "vulnerabilities"
+ ],
+ "additionalProperties": true,
+ "properties": {
+ "scan": {
+ "type": "object",
+ "required": [
+ "analyzer",
+ "end_time",
+ "scanner",
+ "start_time",
+ "status",
+ "type"
+ ],
+ "properties": {
+ "end_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-01-28T03:26:02"
+ ]
+ },
+ "messages": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Communication intended for the initiator of a scan.",
+ "required": [
+ "level",
+ "value"
+ ],
+ "properties": {
+ "level": {
+ "type": "string",
+ "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+ "enum": [
+ "info",
+ "warn",
+ "fatal"
+ ],
+ "examples": [
+ "info"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "The message to communicate.",
+ "minLength": 1,
+ "examples": [
+ "Permission denied, scanning aborted"
+ ]
+ }
+ }
+ }
+ },
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
+ "analyzer": {
+ "type": "object",
+ "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "gitlab-dast"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the analyzer, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "GitLab DAST"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "pattern": "^https?://.+",
+ "description": "A link to more information about the analyzer.",
+ "examples": [
+ "https://docs.gitlab.com/ee/user/application_security/dast"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the analyzer.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ }
+ }
+ },
+ "scanner": {
+ "type": "object",
+ "description": "Object defining the scanner used to perform the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the scanner.",
+ "minLength": 1,
+ "examples": [
+ "my-sast-scanner"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the scanner, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "My SAST Scanner"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "A link to more information about the scanner.",
+ "examples": [
+ "https://scanner.url"
+ ]
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the scanner.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the scanner.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ }
+ }
+ },
+ "start_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-02-14T16:01:59"
+ ]
+ },
+ "status": {
+ "type": "string",
+ "description": "Result of the scan.",
+ "enum": [
+ "success",
+ "failure"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "description": "Type of the scan.",
+ "enum": [
+ "coverage_fuzzing"
+ ]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ }
+ }
+ },
+ "schema": {
+ "type": "string",
+ "description": "URI pointing to the validating security report schema.",
+ "pattern": "^https?://.+"
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the schema to which the JSON report conforms.",
+ "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+ },
+ "vulnerabilities": {
+ "type": "array",
+ "description": "Array of vulnerability objects.",
+ "items": {
+ "type": "object",
+ "description": "Describes the vulnerability using GitLab Flavored Markdown",
+ "required": [
+ "id",
+ "identifiers",
+ "location"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "maxLength": 255,
+ "description": "The name of the vulnerability. This must not include the finding's specific information."
+ },
+ "description": {
+ "type": "string",
+ "maxLength": 1048576,
+ "description": "A long text section describing the vulnerability more fully."
+ },
+ "severity": {
+ "type": "string",
+ "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+ "enum": [
+ "Info",
+ "Unknown",
+ "Low",
+ "Medium",
+ "High",
+ "Critical"
+ ]
+ },
+ "solution": {
+ "type": "string",
+ "maxLength": 7000,
+ "description": "Explanation of how to fix the vulnerability."
+ },
+ "identifiers": {
+ "type": "array",
+ "minItems": 1,
+ "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ },
+ "cvss_vectors": {
+ "type": "array",
+ "minItems": 1,
+ "maxItems": 10,
+ "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.",
+ "items": {
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 16,
+ "maxLength": 128,
+ "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ },
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 32,
+ "maxLength": 128,
+ "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ }
+ ]
+ }
+ },
+ "links": {
+ "type": "array",
+ "description": "An array of references to external documentation or articles that describe the vulnerability.",
+ "items": {
+ "type": "object",
+ "required": [
+ "url"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the vulnerability details link."
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the vulnerability details document.",
+ "pattern": "^(https?|ftp)://.+"
+ }
+ }
+ }
+ },
+ "details": {
+ "$ref": "#/definitions/named_list/properties/items"
+ },
+ "tracking": {
+ "type": "object",
+ "description": "Describes how this vulnerability should be tracked as the project changes.",
+ "oneOf": [
+ {
+ "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
+ "required": [
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "source"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "description": "An item that should be tracked using source-specific tracking methods.",
+ "type": "object",
+ "required": [
+ "signatures"
+ ],
+ "properties": {
+ "file": {
+ "type": "string",
+ "description": "Path to the file where the vulnerability is located."
+ },
+ "start_line": {
+ "type": "number",
+ "description": "The first line of the file that includes the vulnerability."
+ },
+ "end_line": {
+ "type": "number",
+ "description": "The last line of the file that includes the vulnerability."
+ },
+ "signatures": {
+ "type": "array",
+ "description": "An array of calculated tracking signatures for this tracking item.",
+ "minItems": 1,
+ "items": {
+ "description": "A calculated tracking signature value and metadata.",
+ "type": "object",
+ "required": [
+ "algorithm",
+ "value"
+ ],
+ "properties": {
+ "algorithm": {
+ "type": "string",
+ "description": "The algorithm used to generate the signature."
+ },
+ "value": {
+ "type": "string",
+ "description": "The result of this signature algorithm."
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "Each tracking type must declare its own type."
+ }
+ }
+ },
+ "flags": {
+ "description": "Flags that can be attached to vulnerabilities.",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Informational flags identified and assigned to a vulnerability.",
+ "required": [
+ "type",
+ "origin",
+ "description"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Result of the scan.",
+ "enum": [
+ "flagged-as-likely-false-positive"
+ ]
+ },
+ "origin": {
+ "minLength": 1,
+ "description": "Tool that issued the flag.",
+ "type": "string"
+ },
+ "description": {
+ "minLength": 1,
+ "description": "What the flag is about.",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "location": {
+ "description": "The location of the error",
+ "type": "object",
+ "properties": {
+ "crash_address": {
+ "type": "string",
+ "description": "The relative address in memory were the crash occurred.",
+ "examples": [
+ "0xabababab"
+ ]
+ },
+ "stacktrace_snippet": {
+ "type": "string",
+ "description": "The stack trace recorded during fuzzing resulting the crash.",
+ "examples": [
+ "func_a+0xabcd\nfunc_b+0xabcc"
+ ]
+ },
+ "crash_state": {
+ "type": "string",
+ "description": "Minimised and normalized crash stack-trace (called crash_state).",
+ "examples": [
+ "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc"
+ ]
+ },
+ "crash_type": {
+ "type": "string",
+ "description": "Type of the crash.",
+ "examples": [
+ "Heap-Buffer-overflow",
+ "Division-by-zero"
+ ]
+ }
+ }
+ }
+ }
+ }
+ },
+ "remediations": {
+ "type": "array",
+ "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+ "items": {
+ "type": "object",
+ "required": [
+ "fixes",
+ "summary",
+ "diff"
+ ],
+ "properties": {
+ "fixes": {
+ "type": "array",
+ "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+ "items": {
+ "type": "object",
+ "required": [
+ "id"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ }
+ }
+ }
+ },
+ "summary": {
+ "type": "string",
+ "minLength": 1,
+ "description": "An overview of how the vulnerabilities were fixed."
+ },
+ "diff": {
+ "type": "string",
+ "minLength": 1,
+ "description": "A base64-encoded remediation code diff, compatible with git apply."
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dast-report-format.json
new file mode 100644
index 00000000000..8a9519f442f
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dast-report-format.json
@@ -0,0 +1,1380 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json",
+ "title": "Report format for GitLab DAST",
+ "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).",
+ "definitions": {
+ "detail_type": {
+ "oneOf": [
+ {
+ "$ref": "#/definitions/named_list"
+ },
+ {
+ "$ref": "#/definitions/list"
+ },
+ {
+ "$ref": "#/definitions/table"
+ },
+ {
+ "$ref": "#/definitions/text"
+ },
+ {
+ "$ref": "#/definitions/url"
+ },
+ {
+ "$ref": "#/definitions/code"
+ },
+ {
+ "$ref": "#/definitions/value"
+ },
+ {
+ "$ref": "#/definitions/diff"
+ },
+ {
+ "$ref": "#/definitions/markdown"
+ },
+ {
+ "$ref": "#/definitions/commit"
+ },
+ {
+ "$ref": "#/definitions/file_location"
+ },
+ {
+ "$ref": "#/definitions/module_location"
+ }
+ ]
+ },
+ "text_value": {
+ "type": "string"
+ },
+ "named_field": {
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "$ref": "#/definitions/text_value",
+ "type": "string",
+ "minLength": 1
+ },
+ "description": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "named_list": {
+ "type": "object",
+ "description": "An object with named and typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "named-list"
+ },
+ "items": {
+ "type": "object",
+ "patternProperties": {
+ "^.*$": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/named_field"
+ },
+ {
+ "$ref": "#/definitions/detail_type"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "list": {
+ "type": "object",
+ "description": "A list of typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "list"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ },
+ "table": {
+ "type": "object",
+ "description": "A table of typed fields",
+ "required": [
+ "type",
+ "rows"
+ ],
+ "properties": {
+ "type": {
+ "const": "table"
+ },
+ "header": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ },
+ "rows": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ }
+ },
+ "text": {
+ "type": "object",
+ "description": "Raw text",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "text"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "url": {
+ "type": "object",
+ "description": "A single URL",
+ "required": [
+ "type",
+ "href"
+ ],
+ "properties": {
+ "type": {
+ "const": "url"
+ },
+ "text": {
+ "$ref": "#/definitions/text_value"
+ },
+ "href": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "http://mysite.com"
+ ]
+ }
+ }
+ },
+ "code": {
+ "type": "object",
+ "description": "A codeblock",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "code"
+ },
+ "value": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string",
+ "description": "A programming language"
+ }
+ }
+ },
+ "value": {
+ "type": "object",
+ "description": "A field that can store a range of types of value",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "value"
+ },
+ "value": {
+ "type": [
+ "number",
+ "string",
+ "boolean"
+ ]
+ }
+ }
+ },
+ "diff": {
+ "type": "object",
+ "description": "A diff",
+ "required": [
+ "type",
+ "before",
+ "after"
+ ],
+ "properties": {
+ "type": {
+ "const": "diff"
+ },
+ "before": {
+ "type": "string"
+ },
+ "after": {
+ "type": "string"
+ }
+ }
+ },
+ "markdown": {
+ "type": "object",
+ "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "markdown"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value",
+ "examples": [
+ "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+ ]
+ }
+ }
+ },
+ "commit": {
+ "type": "object",
+ "description": "A commit/tag/branch within the GitLab project",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "commit"
+ },
+ "value": {
+ "type": "string",
+ "description": "The commit SHA",
+ "minLength": 1
+ }
+ }
+ },
+ "file_location": {
+ "type": "object",
+ "description": "A location within a file in the project",
+ "required": [
+ "type",
+ "file_name",
+ "line_start"
+ ],
+ "properties": {
+ "type": {
+ "const": "file-location"
+ },
+ "file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "line_start": {
+ "type": "integer"
+ },
+ "line_end": {
+ "type": "integer"
+ }
+ }
+ },
+ "module_location": {
+ "type": "object",
+ "description": "A location within a binary module of the form module+relative_offset",
+ "required": [
+ "type",
+ "module_name",
+ "offset"
+ ],
+ "properties": {
+ "type": {
+ "const": "module-location"
+ },
+ "module_name": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "compiled_binary"
+ ]
+ },
+ "offset": {
+ "type": "integer",
+ "examples": [
+ 100
+ ]
+ }
+ }
+ }
+ },
+ "self": {
+ "version": "15.0.7"
+ },
+ "type": "object",
+ "required": [
+ "scan",
+ "version",
+ "vulnerabilities"
+ ],
+ "additionalProperties": true,
+ "properties": {
+ "scan": {
+ "type": "object",
+ "required": [
+ "analyzer",
+ "end_time",
+ "scanned_resources",
+ "scanner",
+ "start_time",
+ "status",
+ "type"
+ ],
+ "properties": {
+ "end_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-01-28T03:26:02"
+ ]
+ },
+ "messages": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Communication intended for the initiator of a scan.",
+ "required": [
+ "level",
+ "value"
+ ],
+ "properties": {
+ "level": {
+ "type": "string",
+ "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+ "enum": [
+ "info",
+ "warn",
+ "fatal"
+ ],
+ "examples": [
+ "info"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "The message to communicate.",
+ "minLength": 1,
+ "examples": [
+ "Permission denied, scanning aborted"
+ ]
+ }
+ }
+ }
+ },
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
+ "analyzer": {
+ "type": "object",
+ "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "gitlab-dast"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the analyzer, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "GitLab DAST"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "pattern": "^https?://.+",
+ "description": "A link to more information about the analyzer.",
+ "examples": [
+ "https://docs.gitlab.com/ee/user/application_security/dast"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the analyzer.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ }
+ }
+ },
+ "scanner": {
+ "type": "object",
+ "description": "Object defining the scanner used to perform the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the scanner.",
+ "minLength": 1,
+ "examples": [
+ "my-sast-scanner"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the scanner, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "My SAST Scanner"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "A link to more information about the scanner.",
+ "examples": [
+ "https://scanner.url"
+ ]
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the scanner.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the scanner.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ }
+ }
+ },
+ "start_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-02-14T16:01:59"
+ ]
+ },
+ "status": {
+ "type": "string",
+ "description": "Result of the scan.",
+ "enum": [
+ "success",
+ "failure"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "description": "Type of the scan.",
+ "enum": [
+ "dast",
+ "api_fuzzing"
+ ]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ },
+ "scanned_resources": {
+ "type": "array",
+ "description": "The attack surface scanned by DAST.",
+ "items": {
+ "type": "object",
+ "required": [
+ "method",
+ "url",
+ "type"
+ ],
+ "properties": {
+ "method": {
+ "type": "string",
+ "minLength": 1,
+ "description": "HTTP method of the scanned resource.",
+ "examples": [
+ "GET",
+ "POST",
+ "HEAD"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "minLength": 1,
+ "description": "URL of the scanned resource.",
+ "examples": [
+ "http://my.site.com/a-page"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Type of the scanned resource, for DAST, this must be 'url'.",
+ "examples": [
+ "url"
+ ]
+ }
+ }
+ }
+ }
+ }
+ },
+ "schema": {
+ "type": "string",
+ "description": "URI pointing to the validating security report schema.",
+ "pattern": "^https?://.+"
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the schema to which the JSON report conforms.",
+ "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+ },
+ "vulnerabilities": {
+ "type": "array",
+ "description": "Array of vulnerability objects.",
+ "items": {
+ "type": "object",
+ "description": "Describes the vulnerability using GitLab Flavored Markdown",
+ "required": [
+ "id",
+ "identifiers",
+ "location"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "maxLength": 255,
+ "description": "The name of the vulnerability. This must not include the finding's specific information."
+ },
+ "description": {
+ "type": "string",
+ "maxLength": 1048576,
+ "description": "A long text section describing the vulnerability more fully."
+ },
+ "severity": {
+ "type": "string",
+ "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+ "enum": [
+ "Info",
+ "Unknown",
+ "Low",
+ "Medium",
+ "High",
+ "Critical"
+ ]
+ },
+ "solution": {
+ "type": "string",
+ "maxLength": 7000,
+ "description": "Explanation of how to fix the vulnerability."
+ },
+ "identifiers": {
+ "type": "array",
+ "minItems": 1,
+ "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ },
+ "cvss_vectors": {
+ "type": "array",
+ "minItems": 1,
+ "maxItems": 10,
+ "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.",
+ "items": {
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 16,
+ "maxLength": 128,
+ "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ },
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 32,
+ "maxLength": 128,
+ "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ }
+ ]
+ }
+ },
+ "links": {
+ "type": "array",
+ "description": "An array of references to external documentation or articles that describe the vulnerability.",
+ "items": {
+ "type": "object",
+ "required": [
+ "url"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the vulnerability details link."
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the vulnerability details document.",
+ "pattern": "^(https?|ftp)://.+"
+ }
+ }
+ }
+ },
+ "details": {
+ "$ref": "#/definitions/named_list/properties/items"
+ },
+ "tracking": {
+ "type": "object",
+ "description": "Describes how this vulnerability should be tracked as the project changes.",
+ "oneOf": [
+ {
+ "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
+ "required": [
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "source"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "description": "An item that should be tracked using source-specific tracking methods.",
+ "type": "object",
+ "required": [
+ "signatures"
+ ],
+ "properties": {
+ "file": {
+ "type": "string",
+ "description": "Path to the file where the vulnerability is located."
+ },
+ "start_line": {
+ "type": "number",
+ "description": "The first line of the file that includes the vulnerability."
+ },
+ "end_line": {
+ "type": "number",
+ "description": "The last line of the file that includes the vulnerability."
+ },
+ "signatures": {
+ "type": "array",
+ "description": "An array of calculated tracking signatures for this tracking item.",
+ "minItems": 1,
+ "items": {
+ "description": "A calculated tracking signature value and metadata.",
+ "type": "object",
+ "required": [
+ "algorithm",
+ "value"
+ ],
+ "properties": {
+ "algorithm": {
+ "type": "string",
+ "description": "The algorithm used to generate the signature."
+ },
+ "value": {
+ "type": "string",
+ "description": "The result of this signature algorithm."
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "Each tracking type must declare its own type."
+ }
+ }
+ },
+ "flags": {
+ "description": "Flags that can be attached to vulnerabilities.",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Informational flags identified and assigned to a vulnerability.",
+ "required": [
+ "type",
+ "origin",
+ "description"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Result of the scan.",
+ "enum": [
+ "flagged-as-likely-false-positive"
+ ]
+ },
+ "origin": {
+ "minLength": 1,
+ "description": "Tool that issued the flag.",
+ "type": "string"
+ },
+ "description": {
+ "minLength": 1,
+ "description": "What the flag is about.",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "evidence": {
+ "type": "object",
+ "properties": {
+ "source": {
+ "type": "object",
+ "description": "Source of evidence",
+ "required": [
+ "id",
+ "name"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique source identifier",
+ "examples": [
+ "assert:LogAnalysis",
+ "assert:StatusCode"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Source display name",
+ "examples": [
+ "Log Analysis",
+ "Status Code"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "Link to additional information",
+ "examples": [
+ "https://docs.gitlab.com/ee/development/integrations/secure.html"
+ ]
+ }
+ }
+ },
+ "summary": {
+ "type": "string",
+ "description": "Human readable string containing evidence of the vulnerability.",
+ "examples": [
+ "Credit card 4111111111111111 found",
+ "Server leaked information nginx/1.17.6"
+ ]
+ },
+ "request": {
+ "type": "object",
+ "description": "An HTTP request.",
+ "required": [
+ "headers",
+ "method",
+ "url"
+ ],
+ "properties": {
+ "headers": {
+ "type": "array",
+ "description": "HTTP headers present on the request.",
+ "items": {
+ "type": "object",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Name of the HTTP header.",
+ "examples": [
+ "Accept",
+ "Content-Length",
+ "Content-Type"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the HTTP header.",
+ "examples": [
+ "*/*",
+ "560",
+ "application/json; charset=utf-8"
+ ]
+ }
+ }
+ }
+ },
+ "method": {
+ "type": "string",
+ "minLength": 1,
+ "description": "HTTP method used in the request.",
+ "examples": [
+ "GET",
+ "POST"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "minLength": 1,
+ "description": "URL of the request.",
+ "examples": [
+ "http://my.site.com/vulnerable-endpoint?show-credit-card"
+ ]
+ },
+ "body": {
+ "type": "string",
+ "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
+ "examples": [
+ "user=jsmith&first=%27&last=smith"
+ ]
+ }
+ }
+ },
+ "response": {
+ "type": "object",
+ "description": "An HTTP response.",
+ "required": [
+ "headers",
+ "reason_phrase",
+ "status_code"
+ ],
+ "properties": {
+ "headers": {
+ "type": "array",
+ "description": "HTTP headers present on the request.",
+ "items": {
+ "type": "object",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Name of the HTTP header.",
+ "examples": [
+ "Accept",
+ "Content-Length",
+ "Content-Type"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the HTTP header.",
+ "examples": [
+ "*/*",
+ "560",
+ "application/json; charset=utf-8"
+ ]
+ }
+ }
+ }
+ },
+ "reason_phrase": {
+ "type": "string",
+ "description": "HTTP reason phrase of the response.",
+ "examples": [
+ "OK",
+ "Internal Server Error"
+ ]
+ },
+ "status_code": {
+ "type": "integer",
+ "description": "HTTP status code of the response.",
+ "examples": [
+ 200,
+ 500
+ ]
+ },
+ "body": {
+ "type": "string",
+ "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
+ "examples": [
+ "{\"user_id\": 2}"
+ ]
+ }
+ }
+ },
+ "supporting_messages": {
+ "type": "array",
+ "description": "Array of supporting http messages.",
+ "items": {
+ "type": "object",
+ "description": "A supporting http message.",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Message display name.",
+ "examples": [
+ "Unmodified",
+ "Recorded"
+ ]
+ },
+ "request": {
+ "type": "object",
+ "description": "An HTTP request.",
+ "required": [
+ "headers",
+ "method",
+ "url"
+ ],
+ "properties": {
+ "headers": {
+ "type": "array",
+ "description": "HTTP headers present on the request.",
+ "items": {
+ "type": "object",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Name of the HTTP header.",
+ "examples": [
+ "Accept",
+ "Content-Length",
+ "Content-Type"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the HTTP header.",
+ "examples": [
+ "*/*",
+ "560",
+ "application/json; charset=utf-8"
+ ]
+ }
+ }
+ }
+ },
+ "method": {
+ "type": "string",
+ "minLength": 1,
+ "description": "HTTP method used in the request.",
+ "examples": [
+ "GET",
+ "POST"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "minLength": 1,
+ "description": "URL of the request.",
+ "examples": [
+ "http://my.site.com/vulnerable-endpoint?show-credit-card"
+ ]
+ },
+ "body": {
+ "type": "string",
+ "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
+ "examples": [
+ "user=jsmith&first=%27&last=smith"
+ ]
+ }
+ }
+ },
+ "response": {
+ "type": "object",
+ "description": "An HTTP response.",
+ "required": [
+ "headers",
+ "reason_phrase",
+ "status_code"
+ ],
+ "properties": {
+ "headers": {
+ "type": "array",
+ "description": "HTTP headers present on the request.",
+ "items": {
+ "type": "object",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Name of the HTTP header.",
+ "examples": [
+ "Accept",
+ "Content-Length",
+ "Content-Type"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the HTTP header.",
+ "examples": [
+ "*/*",
+ "560",
+ "application/json; charset=utf-8"
+ ]
+ }
+ }
+ }
+ },
+ "reason_phrase": {
+ "type": "string",
+ "description": "HTTP reason phrase of the response.",
+ "examples": [
+ "OK",
+ "Internal Server Error"
+ ]
+ },
+ "status_code": {
+ "type": "integer",
+ "description": "HTTP status code of the response.",
+ "examples": [
+ 200,
+ 500
+ ]
+ },
+ "body": {
+ "type": "string",
+ "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.",
+ "examples": [
+ "{\"user_id\": 2}"
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "location": {
+ "type": "object",
+ "description": "Identifies the vulnerability's location.",
+ "properties": {
+ "hostname": {
+ "type": "string",
+ "description": "The protocol, domain, and port of the application where the vulnerability was found."
+ },
+ "method": {
+ "type": "string",
+ "description": "The HTTP method that was used to request the URL where the vulnerability was found."
+ },
+ "param": {
+ "type": "string",
+ "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST."
+ },
+ "path": {
+ "type": "string",
+ "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash."
+ }
+ }
+ },
+ "assets": {
+ "type": "array",
+ "description": "Array of build assets associated with vulnerability.",
+ "items": {
+ "type": "object",
+ "description": "Describes an asset associated with vulnerability.",
+ "required": [
+ "type",
+ "name",
+ "url"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "The type of asset",
+ "enum": [
+ "http_session",
+ "postman"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Display name for asset",
+ "examples": [
+ "HTTP Messages",
+ "Postman Collection"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Link to asset in build artifacts",
+ "examples": [
+ "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data"
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "remediations": {
+ "type": "array",
+ "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+ "items": {
+ "type": "object",
+ "required": [
+ "fixes",
+ "summary",
+ "diff"
+ ],
+ "properties": {
+ "fixes": {
+ "type": "array",
+ "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+ "items": {
+ "type": "object",
+ "required": [
+ "id"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ }
+ }
+ }
+ },
+ "summary": {
+ "type": "string",
+ "minLength": 1,
+ "description": "An overview of how the vulnerabilities were fixed."
+ },
+ "diff": {
+ "type": "string",
+ "minLength": 1,
+ "description": "A base64-encoded remediation code diff, compatible with git apply."
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dependency-scanning-report-format.json
new file mode 100644
index 00000000000..83b3537b5f1
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dependency-scanning-report-format.json
@@ -0,0 +1,1083 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json",
+ "title": "Report format for GitLab Dependency Scanning",
+ "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).",
+ "definitions": {
+ "detail_type": {
+ "oneOf": [
+ {
+ "$ref": "#/definitions/named_list"
+ },
+ {
+ "$ref": "#/definitions/list"
+ },
+ {
+ "$ref": "#/definitions/table"
+ },
+ {
+ "$ref": "#/definitions/text"
+ },
+ {
+ "$ref": "#/definitions/url"
+ },
+ {
+ "$ref": "#/definitions/code"
+ },
+ {
+ "$ref": "#/definitions/value"
+ },
+ {
+ "$ref": "#/definitions/diff"
+ },
+ {
+ "$ref": "#/definitions/markdown"
+ },
+ {
+ "$ref": "#/definitions/commit"
+ },
+ {
+ "$ref": "#/definitions/file_location"
+ },
+ {
+ "$ref": "#/definitions/module_location"
+ }
+ ]
+ },
+ "text_value": {
+ "type": "string"
+ },
+ "named_field": {
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "$ref": "#/definitions/text_value",
+ "type": "string",
+ "minLength": 1
+ },
+ "description": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "named_list": {
+ "type": "object",
+ "description": "An object with named and typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "named-list"
+ },
+ "items": {
+ "type": "object",
+ "patternProperties": {
+ "^.*$": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/named_field"
+ },
+ {
+ "$ref": "#/definitions/detail_type"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "list": {
+ "type": "object",
+ "description": "A list of typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "list"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ },
+ "table": {
+ "type": "object",
+ "description": "A table of typed fields",
+ "required": [
+ "type",
+ "rows"
+ ],
+ "properties": {
+ "type": {
+ "const": "table"
+ },
+ "header": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ },
+ "rows": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ }
+ },
+ "text": {
+ "type": "object",
+ "description": "Raw text",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "text"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "url": {
+ "type": "object",
+ "description": "A single URL",
+ "required": [
+ "type",
+ "href"
+ ],
+ "properties": {
+ "type": {
+ "const": "url"
+ },
+ "text": {
+ "$ref": "#/definitions/text_value"
+ },
+ "href": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "http://mysite.com"
+ ]
+ }
+ }
+ },
+ "code": {
+ "type": "object",
+ "description": "A codeblock",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "code"
+ },
+ "value": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string",
+ "description": "A programming language"
+ }
+ }
+ },
+ "value": {
+ "type": "object",
+ "description": "A field that can store a range of types of value",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "value"
+ },
+ "value": {
+ "type": [
+ "number",
+ "string",
+ "boolean"
+ ]
+ }
+ }
+ },
+ "diff": {
+ "type": "object",
+ "description": "A diff",
+ "required": [
+ "type",
+ "before",
+ "after"
+ ],
+ "properties": {
+ "type": {
+ "const": "diff"
+ },
+ "before": {
+ "type": "string"
+ },
+ "after": {
+ "type": "string"
+ }
+ }
+ },
+ "markdown": {
+ "type": "object",
+ "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "markdown"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value",
+ "examples": [
+ "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+ ]
+ }
+ }
+ },
+ "commit": {
+ "type": "object",
+ "description": "A commit/tag/branch within the GitLab project",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "commit"
+ },
+ "value": {
+ "type": "string",
+ "description": "The commit SHA",
+ "minLength": 1
+ }
+ }
+ },
+ "file_location": {
+ "type": "object",
+ "description": "A location within a file in the project",
+ "required": [
+ "type",
+ "file_name",
+ "line_start"
+ ],
+ "properties": {
+ "type": {
+ "const": "file-location"
+ },
+ "file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "line_start": {
+ "type": "integer"
+ },
+ "line_end": {
+ "type": "integer"
+ }
+ }
+ },
+ "module_location": {
+ "type": "object",
+ "description": "A location within a binary module of the form module+relative_offset",
+ "required": [
+ "type",
+ "module_name",
+ "offset"
+ ],
+ "properties": {
+ "type": {
+ "const": "module-location"
+ },
+ "module_name": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "compiled_binary"
+ ]
+ },
+ "offset": {
+ "type": "integer",
+ "examples": [
+ 100
+ ]
+ }
+ }
+ }
+ },
+ "self": {
+ "version": "15.0.7"
+ },
+ "type": "object",
+ "required": [
+ "dependency_files",
+ "scan",
+ "version",
+ "vulnerabilities"
+ ],
+ "additionalProperties": true,
+ "properties": {
+ "scan": {
+ "type": "object",
+ "required": [
+ "analyzer",
+ "end_time",
+ "scanner",
+ "start_time",
+ "status",
+ "type"
+ ],
+ "properties": {
+ "end_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-01-28T03:26:02"
+ ]
+ },
+ "messages": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Communication intended for the initiator of a scan.",
+ "required": [
+ "level",
+ "value"
+ ],
+ "properties": {
+ "level": {
+ "type": "string",
+ "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+ "enum": [
+ "info",
+ "warn",
+ "fatal"
+ ],
+ "examples": [
+ "info"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "The message to communicate.",
+ "minLength": 1,
+ "examples": [
+ "Permission denied, scanning aborted"
+ ]
+ }
+ }
+ }
+ },
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
+ "analyzer": {
+ "type": "object",
+ "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "gitlab-dast"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the analyzer, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "GitLab DAST"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "pattern": "^https?://.+",
+ "description": "A link to more information about the analyzer.",
+ "examples": [
+ "https://docs.gitlab.com/ee/user/application_security/dast"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the analyzer.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ }
+ }
+ },
+ "scanner": {
+ "type": "object",
+ "description": "Object defining the scanner used to perform the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the scanner.",
+ "minLength": 1,
+ "examples": [
+ "my-sast-scanner"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the scanner, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "My SAST Scanner"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "A link to more information about the scanner.",
+ "examples": [
+ "https://scanner.url"
+ ]
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the scanner.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the scanner.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ }
+ }
+ },
+ "start_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-02-14T16:01:59"
+ ]
+ },
+ "status": {
+ "type": "string",
+ "description": "Result of the scan.",
+ "enum": [
+ "success",
+ "failure"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "description": "Type of the scan.",
+ "enum": [
+ "dependency_scanning"
+ ]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ }
+ }
+ },
+ "schema": {
+ "type": "string",
+ "description": "URI pointing to the validating security report schema.",
+ "pattern": "^https?://.+"
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the schema to which the JSON report conforms.",
+ "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+ },
+ "vulnerabilities": {
+ "type": "array",
+ "description": "Array of vulnerability objects.",
+ "items": {
+ "type": "object",
+ "description": "Describes the vulnerability using GitLab Flavored Markdown",
+ "required": [
+ "id",
+ "identifiers",
+ "location"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "maxLength": 255,
+ "description": "The name of the vulnerability. This must not include the finding's specific information."
+ },
+ "description": {
+ "type": "string",
+ "maxLength": 1048576,
+ "description": "A long text section describing the vulnerability more fully."
+ },
+ "severity": {
+ "type": "string",
+ "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+ "enum": [
+ "Info",
+ "Unknown",
+ "Low",
+ "Medium",
+ "High",
+ "Critical"
+ ]
+ },
+ "solution": {
+ "type": "string",
+ "maxLength": 7000,
+ "description": "Explanation of how to fix the vulnerability."
+ },
+ "identifiers": {
+ "type": "array",
+ "minItems": 1,
+ "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ },
+ "cvss_vectors": {
+ "type": "array",
+ "minItems": 1,
+ "maxItems": 10,
+ "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.",
+ "items": {
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 16,
+ "maxLength": 128,
+ "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ },
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 32,
+ "maxLength": 128,
+ "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ }
+ ]
+ }
+ },
+ "links": {
+ "type": "array",
+ "description": "An array of references to external documentation or articles that describe the vulnerability.",
+ "items": {
+ "type": "object",
+ "required": [
+ "url"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the vulnerability details link."
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the vulnerability details document.",
+ "pattern": "^(https?|ftp)://.+"
+ }
+ }
+ }
+ },
+ "details": {
+ "$ref": "#/definitions/named_list/properties/items"
+ },
+ "tracking": {
+ "type": "object",
+ "description": "Describes how this vulnerability should be tracked as the project changes.",
+ "oneOf": [
+ {
+ "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
+ "required": [
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "source"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "description": "An item that should be tracked using source-specific tracking methods.",
+ "type": "object",
+ "required": [
+ "signatures"
+ ],
+ "properties": {
+ "file": {
+ "type": "string",
+ "description": "Path to the file where the vulnerability is located."
+ },
+ "start_line": {
+ "type": "number",
+ "description": "The first line of the file that includes the vulnerability."
+ },
+ "end_line": {
+ "type": "number",
+ "description": "The last line of the file that includes the vulnerability."
+ },
+ "signatures": {
+ "type": "array",
+ "description": "An array of calculated tracking signatures for this tracking item.",
+ "minItems": 1,
+ "items": {
+ "description": "A calculated tracking signature value and metadata.",
+ "type": "object",
+ "required": [
+ "algorithm",
+ "value"
+ ],
+ "properties": {
+ "algorithm": {
+ "type": "string",
+ "description": "The algorithm used to generate the signature."
+ },
+ "value": {
+ "type": "string",
+ "description": "The result of this signature algorithm."
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "Each tracking type must declare its own type."
+ }
+ }
+ },
+ "flags": {
+ "description": "Flags that can be attached to vulnerabilities.",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Informational flags identified and assigned to a vulnerability.",
+ "required": [
+ "type",
+ "origin",
+ "description"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Result of the scan.",
+ "enum": [
+ "flagged-as-likely-false-positive"
+ ]
+ },
+ "origin": {
+ "minLength": 1,
+ "description": "Tool that issued the flag.",
+ "type": "string"
+ },
+ "description": {
+ "minLength": 1,
+ "description": "What the flag is about.",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "location": {
+ "type": "object",
+ "description": "Identifies the vulnerability's location.",
+ "required": [
+ "file",
+ "dependency"
+ ],
+ "properties": {
+ "file": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)."
+ },
+ "dependency": {
+ "type": "object",
+ "description": "Describes the dependency of a project where the vulnerability is located.",
+ "required": [
+ "package",
+ "version"
+ ],
+ "properties": {
+ "package": {
+ "type": "object",
+ "description": "Provides information on the package where the vulnerability is located.",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the package where the vulnerability is located."
+ }
+ }
+ },
+ "version": {
+ "type": "string",
+ "description": "Version of the vulnerable package."
+ },
+ "iid": {
+ "description": "ID that identifies the dependency in the scope of a dependency file.",
+ "type": "number"
+ },
+ "direct": {
+ "type": "boolean",
+ "description": "Tells whether this is a direct, top-level dependency of the scanned project."
+ },
+ "dependency_path": {
+ "type": "array",
+ "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
+ "items": {
+ "type": "object",
+ "required": [
+ "iid"
+ ],
+ "properties": {
+ "iid": {
+ "type": "number",
+ "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "remediations": {
+ "type": "array",
+ "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+ "items": {
+ "type": "object",
+ "required": [
+ "fixes",
+ "summary",
+ "diff"
+ ],
+ "properties": {
+ "fixes": {
+ "type": "array",
+ "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+ "items": {
+ "type": "object",
+ "required": [
+ "id"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ }
+ }
+ }
+ },
+ "summary": {
+ "type": "string",
+ "minLength": 1,
+ "description": "An overview of how the vulnerabilities were fixed."
+ },
+ "diff": {
+ "type": "string",
+ "minLength": 1,
+ "description": "A base64-encoded remediation code diff, compatible with git apply."
+ }
+ }
+ }
+ },
+ "dependency_files": {
+ "type": "array",
+ "description": "List of dependency files identified in the project.",
+ "items": {
+ "type": "object",
+ "required": [
+ "path",
+ "package_manager",
+ "dependencies"
+ ],
+ "properties": {
+ "path": {
+ "type": "string",
+ "minLength": 1
+ },
+ "package_manager": {
+ "type": "string",
+ "minLength": 1
+ },
+ "dependencies": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Describes the dependency of a project where the vulnerability is located.",
+ "required": [
+ "package",
+ "version"
+ ],
+ "properties": {
+ "package": {
+ "type": "object",
+ "description": "Provides information on the package where the vulnerability is located.",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the package where the vulnerability is located."
+ }
+ }
+ },
+ "version": {
+ "type": "string",
+ "description": "Version of the vulnerable package."
+ },
+ "iid": {
+ "description": "ID that identifies the dependency in the scope of a dependency file.",
+ "type": "number"
+ },
+ "direct": {
+ "type": "boolean",
+ "description": "Tells whether this is a direct, top-level dependency of the scanned project."
+ },
+ "dependency_path": {
+ "type": "array",
+ "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.",
+ "items": {
+ "type": "object",
+ "required": [
+ "iid"
+ ],
+ "properties": {
+ "iid": {
+ "type": "number",
+ "description": "ID that is unique in the scope of a parent object, and specific to the resource type."
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/sast-report-format.json
new file mode 100644
index 00000000000..3597ed169d5
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/sast-report-format.json
@@ -0,0 +1,970 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json",
+ "title": "Report format for GitLab SAST",
+ "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
+ "definitions": {
+ "detail_type": {
+ "oneOf": [
+ {
+ "$ref": "#/definitions/named_list"
+ },
+ {
+ "$ref": "#/definitions/list"
+ },
+ {
+ "$ref": "#/definitions/table"
+ },
+ {
+ "$ref": "#/definitions/text"
+ },
+ {
+ "$ref": "#/definitions/url"
+ },
+ {
+ "$ref": "#/definitions/code"
+ },
+ {
+ "$ref": "#/definitions/value"
+ },
+ {
+ "$ref": "#/definitions/diff"
+ },
+ {
+ "$ref": "#/definitions/markdown"
+ },
+ {
+ "$ref": "#/definitions/commit"
+ },
+ {
+ "$ref": "#/definitions/file_location"
+ },
+ {
+ "$ref": "#/definitions/module_location"
+ }
+ ]
+ },
+ "text_value": {
+ "type": "string"
+ },
+ "named_field": {
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "$ref": "#/definitions/text_value",
+ "type": "string",
+ "minLength": 1
+ },
+ "description": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "named_list": {
+ "type": "object",
+ "description": "An object with named and typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "named-list"
+ },
+ "items": {
+ "type": "object",
+ "patternProperties": {
+ "^.*$": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/named_field"
+ },
+ {
+ "$ref": "#/definitions/detail_type"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "list": {
+ "type": "object",
+ "description": "A list of typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "list"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ },
+ "table": {
+ "type": "object",
+ "description": "A table of typed fields",
+ "required": [
+ "type",
+ "rows"
+ ],
+ "properties": {
+ "type": {
+ "const": "table"
+ },
+ "header": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ },
+ "rows": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ }
+ },
+ "text": {
+ "type": "object",
+ "description": "Raw text",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "text"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "url": {
+ "type": "object",
+ "description": "A single URL",
+ "required": [
+ "type",
+ "href"
+ ],
+ "properties": {
+ "type": {
+ "const": "url"
+ },
+ "text": {
+ "$ref": "#/definitions/text_value"
+ },
+ "href": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "http://mysite.com"
+ ]
+ }
+ }
+ },
+ "code": {
+ "type": "object",
+ "description": "A codeblock",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "code"
+ },
+ "value": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string",
+ "description": "A programming language"
+ }
+ }
+ },
+ "value": {
+ "type": "object",
+ "description": "A field that can store a range of types of value",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "value"
+ },
+ "value": {
+ "type": [
+ "number",
+ "string",
+ "boolean"
+ ]
+ }
+ }
+ },
+ "diff": {
+ "type": "object",
+ "description": "A diff",
+ "required": [
+ "type",
+ "before",
+ "after"
+ ],
+ "properties": {
+ "type": {
+ "const": "diff"
+ },
+ "before": {
+ "type": "string"
+ },
+ "after": {
+ "type": "string"
+ }
+ }
+ },
+ "markdown": {
+ "type": "object",
+ "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "markdown"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value",
+ "examples": [
+ "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+ ]
+ }
+ }
+ },
+ "commit": {
+ "type": "object",
+ "description": "A commit/tag/branch within the GitLab project",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "commit"
+ },
+ "value": {
+ "type": "string",
+ "description": "The commit SHA",
+ "minLength": 1
+ }
+ }
+ },
+ "file_location": {
+ "type": "object",
+ "description": "A location within a file in the project",
+ "required": [
+ "type",
+ "file_name",
+ "line_start"
+ ],
+ "properties": {
+ "type": {
+ "const": "file-location"
+ },
+ "file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "line_start": {
+ "type": "integer"
+ },
+ "line_end": {
+ "type": "integer"
+ }
+ }
+ },
+ "module_location": {
+ "type": "object",
+ "description": "A location within a binary module of the form module+relative_offset",
+ "required": [
+ "type",
+ "module_name",
+ "offset"
+ ],
+ "properties": {
+ "type": {
+ "const": "module-location"
+ },
+ "module_name": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "compiled_binary"
+ ]
+ },
+ "offset": {
+ "type": "integer",
+ "examples": [
+ 100
+ ]
+ }
+ }
+ }
+ },
+ "self": {
+ "version": "15.0.7"
+ },
+ "type": "object",
+ "required": [
+ "scan",
+ "version",
+ "vulnerabilities"
+ ],
+ "additionalProperties": true,
+ "properties": {
+ "scan": {
+ "type": "object",
+ "required": [
+ "analyzer",
+ "end_time",
+ "scanner",
+ "start_time",
+ "status",
+ "type"
+ ],
+ "properties": {
+ "end_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-01-28T03:26:02"
+ ]
+ },
+ "messages": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Communication intended for the initiator of a scan.",
+ "required": [
+ "level",
+ "value"
+ ],
+ "properties": {
+ "level": {
+ "type": "string",
+ "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+ "enum": [
+ "info",
+ "warn",
+ "fatal"
+ ],
+ "examples": [
+ "info"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "The message to communicate.",
+ "minLength": 1,
+ "examples": [
+ "Permission denied, scanning aborted"
+ ]
+ }
+ }
+ }
+ },
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
+ "analyzer": {
+ "type": "object",
+ "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "gitlab-dast"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the analyzer, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "GitLab DAST"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "pattern": "^https?://.+",
+ "description": "A link to more information about the analyzer.",
+ "examples": [
+ "https://docs.gitlab.com/ee/user/application_security/dast"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the analyzer.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ }
+ }
+ },
+ "scanner": {
+ "type": "object",
+ "description": "Object defining the scanner used to perform the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the scanner.",
+ "minLength": 1,
+ "examples": [
+ "my-sast-scanner"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the scanner, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "My SAST Scanner"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "A link to more information about the scanner.",
+ "examples": [
+ "https://scanner.url"
+ ]
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the scanner.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the scanner.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ }
+ }
+ },
+ "start_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-02-14T16:01:59"
+ ]
+ },
+ "status": {
+ "type": "string",
+ "description": "Result of the scan.",
+ "enum": [
+ "success",
+ "failure"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "description": "Type of the scan.",
+ "enum": [
+ "sast"
+ ]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ }
+ }
+ },
+ "schema": {
+ "type": "string",
+ "description": "URI pointing to the validating security report schema.",
+ "pattern": "^https?://.+"
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the schema to which the JSON report conforms.",
+ "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+ },
+ "vulnerabilities": {
+ "type": "array",
+ "description": "Array of vulnerability objects.",
+ "items": {
+ "type": "object",
+ "description": "Describes the vulnerability using GitLab Flavored Markdown",
+ "required": [
+ "id",
+ "identifiers",
+ "location"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "maxLength": 255,
+ "description": "The name of the vulnerability. This must not include the finding's specific information."
+ },
+ "description": {
+ "type": "string",
+ "maxLength": 1048576,
+ "description": "A long text section describing the vulnerability more fully."
+ },
+ "severity": {
+ "type": "string",
+ "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+ "enum": [
+ "Info",
+ "Unknown",
+ "Low",
+ "Medium",
+ "High",
+ "Critical"
+ ]
+ },
+ "solution": {
+ "type": "string",
+ "maxLength": 7000,
+ "description": "Explanation of how to fix the vulnerability."
+ },
+ "identifiers": {
+ "type": "array",
+ "minItems": 1,
+ "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ },
+ "cvss_vectors": {
+ "type": "array",
+ "minItems": 1,
+ "maxItems": 10,
+ "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.",
+ "items": {
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 16,
+ "maxLength": 128,
+ "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ },
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 32,
+ "maxLength": 128,
+ "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ }
+ ]
+ }
+ },
+ "links": {
+ "type": "array",
+ "description": "An array of references to external documentation or articles that describe the vulnerability.",
+ "items": {
+ "type": "object",
+ "required": [
+ "url"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the vulnerability details link."
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the vulnerability details document.",
+ "pattern": "^(https?|ftp)://.+"
+ }
+ }
+ }
+ },
+ "details": {
+ "$ref": "#/definitions/named_list/properties/items"
+ },
+ "tracking": {
+ "type": "object",
+ "description": "Describes how this vulnerability should be tracked as the project changes.",
+ "oneOf": [
+ {
+ "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
+ "required": [
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "source"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "description": "An item that should be tracked using source-specific tracking methods.",
+ "type": "object",
+ "required": [
+ "signatures"
+ ],
+ "properties": {
+ "file": {
+ "type": "string",
+ "description": "Path to the file where the vulnerability is located."
+ },
+ "start_line": {
+ "type": "number",
+ "description": "The first line of the file that includes the vulnerability."
+ },
+ "end_line": {
+ "type": "number",
+ "description": "The last line of the file that includes the vulnerability."
+ },
+ "signatures": {
+ "type": "array",
+ "description": "An array of calculated tracking signatures for this tracking item.",
+ "minItems": 1,
+ "items": {
+ "description": "A calculated tracking signature value and metadata.",
+ "type": "object",
+ "required": [
+ "algorithm",
+ "value"
+ ],
+ "properties": {
+ "algorithm": {
+ "type": "string",
+ "description": "The algorithm used to generate the signature."
+ },
+ "value": {
+ "type": "string",
+ "description": "The result of this signature algorithm."
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "Each tracking type must declare its own type."
+ }
+ }
+ },
+ "flags": {
+ "description": "Flags that can be attached to vulnerabilities.",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Informational flags identified and assigned to a vulnerability.",
+ "required": [
+ "type",
+ "origin",
+ "description"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Result of the scan.",
+ "enum": [
+ "flagged-as-likely-false-positive"
+ ]
+ },
+ "origin": {
+ "minLength": 1,
+ "description": "Tool that issued the flag.",
+ "type": "string"
+ },
+ "description": {
+ "minLength": 1,
+ "description": "What the flag is about.",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "location": {
+ "type": "object",
+ "description": "Identifies the vulnerability's location.",
+ "properties": {
+ "file": {
+ "type": "string",
+ "description": "Path to the file where the vulnerability is located."
+ },
+ "start_line": {
+ "type": "number",
+ "description": "The first line of the code affected by the vulnerability."
+ },
+ "end_line": {
+ "type": "number",
+ "description": "The last line of the code affected by the vulnerability."
+ },
+ "class": {
+ "type": "string",
+ "description": "Provides the name of the class where the vulnerability is located."
+ },
+ "method": {
+ "type": "string",
+ "description": "Provides the name of the method where the vulnerability is located."
+ }
+ }
+ },
+ "raw_source_code_extract": {
+ "type": "string",
+ "description": "Provides an unsanitized excerpt of the affected source code."
+ }
+ }
+ }
+ },
+ "remediations": {
+ "type": "array",
+ "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+ "items": {
+ "type": "object",
+ "required": [
+ "fixes",
+ "summary",
+ "diff"
+ ],
+ "properties": {
+ "fixes": {
+ "type": "array",
+ "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+ "items": {
+ "type": "object",
+ "required": [
+ "id"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ }
+ }
+ }
+ },
+ "summary": {
+ "type": "string",
+ "minLength": 1,
+ "description": "An overview of how the vulnerabilities were fixed."
+ },
+ "diff": {
+ "type": "string",
+ "minLength": 1,
+ "description": "A base64-encoded remediation code diff, compatible with git apply."
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/secret-detection-report-format.json
new file mode 100644
index 00000000000..afd80ca916b
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/secret-detection-report-format.json
@@ -0,0 +1,994 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json",
+ "title": "Report format for GitLab Secret Detection",
+ "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
+ "definitions": {
+ "detail_type": {
+ "oneOf": [
+ {
+ "$ref": "#/definitions/named_list"
+ },
+ {
+ "$ref": "#/definitions/list"
+ },
+ {
+ "$ref": "#/definitions/table"
+ },
+ {
+ "$ref": "#/definitions/text"
+ },
+ {
+ "$ref": "#/definitions/url"
+ },
+ {
+ "$ref": "#/definitions/code"
+ },
+ {
+ "$ref": "#/definitions/value"
+ },
+ {
+ "$ref": "#/definitions/diff"
+ },
+ {
+ "$ref": "#/definitions/markdown"
+ },
+ {
+ "$ref": "#/definitions/commit"
+ },
+ {
+ "$ref": "#/definitions/file_location"
+ },
+ {
+ "$ref": "#/definitions/module_location"
+ }
+ ]
+ },
+ "text_value": {
+ "type": "string"
+ },
+ "named_field": {
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "$ref": "#/definitions/text_value",
+ "type": "string",
+ "minLength": 1
+ },
+ "description": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "named_list": {
+ "type": "object",
+ "description": "An object with named and typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "named-list"
+ },
+ "items": {
+ "type": "object",
+ "patternProperties": {
+ "^.*$": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/named_field"
+ },
+ {
+ "$ref": "#/definitions/detail_type"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "list": {
+ "type": "object",
+ "description": "A list of typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "list"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ },
+ "table": {
+ "type": "object",
+ "description": "A table of typed fields",
+ "required": [
+ "type",
+ "rows"
+ ],
+ "properties": {
+ "type": {
+ "const": "table"
+ },
+ "header": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ },
+ "rows": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ }
+ },
+ "text": {
+ "type": "object",
+ "description": "Raw text",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "text"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "url": {
+ "type": "object",
+ "description": "A single URL",
+ "required": [
+ "type",
+ "href"
+ ],
+ "properties": {
+ "type": {
+ "const": "url"
+ },
+ "text": {
+ "$ref": "#/definitions/text_value"
+ },
+ "href": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "http://mysite.com"
+ ]
+ }
+ }
+ },
+ "code": {
+ "type": "object",
+ "description": "A codeblock",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "code"
+ },
+ "value": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string",
+ "description": "A programming language"
+ }
+ }
+ },
+ "value": {
+ "type": "object",
+ "description": "A field that can store a range of types of value",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "value"
+ },
+ "value": {
+ "type": [
+ "number",
+ "string",
+ "boolean"
+ ]
+ }
+ }
+ },
+ "diff": {
+ "type": "object",
+ "description": "A diff",
+ "required": [
+ "type",
+ "before",
+ "after"
+ ],
+ "properties": {
+ "type": {
+ "const": "diff"
+ },
+ "before": {
+ "type": "string"
+ },
+ "after": {
+ "type": "string"
+ }
+ }
+ },
+ "markdown": {
+ "type": "object",
+ "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "markdown"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value",
+ "examples": [
+ "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+ ]
+ }
+ }
+ },
+ "commit": {
+ "type": "object",
+ "description": "A commit/tag/branch within the GitLab project",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "commit"
+ },
+ "value": {
+ "type": "string",
+ "description": "The commit SHA",
+ "minLength": 1
+ }
+ }
+ },
+ "file_location": {
+ "type": "object",
+ "description": "A location within a file in the project",
+ "required": [
+ "type",
+ "file_name",
+ "line_start"
+ ],
+ "properties": {
+ "type": {
+ "const": "file-location"
+ },
+ "file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "line_start": {
+ "type": "integer"
+ },
+ "line_end": {
+ "type": "integer"
+ }
+ }
+ },
+ "module_location": {
+ "type": "object",
+ "description": "A location within a binary module of the form module+relative_offset",
+ "required": [
+ "type",
+ "module_name",
+ "offset"
+ ],
+ "properties": {
+ "type": {
+ "const": "module-location"
+ },
+ "module_name": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "compiled_binary"
+ ]
+ },
+ "offset": {
+ "type": "integer",
+ "examples": [
+ 100
+ ]
+ }
+ }
+ }
+ },
+ "self": {
+ "version": "15.0.7"
+ },
+ "type": "object",
+ "required": [
+ "scan",
+ "version",
+ "vulnerabilities"
+ ],
+ "additionalProperties": true,
+ "properties": {
+ "scan": {
+ "type": "object",
+ "required": [
+ "analyzer",
+ "end_time",
+ "scanner",
+ "start_time",
+ "status",
+ "type"
+ ],
+ "properties": {
+ "end_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-01-28T03:26:02"
+ ]
+ },
+ "messages": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Communication intended for the initiator of a scan.",
+ "required": [
+ "level",
+ "value"
+ ],
+ "properties": {
+ "level": {
+ "type": "string",
+ "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+ "enum": [
+ "info",
+ "warn",
+ "fatal"
+ ],
+ "examples": [
+ "info"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "The message to communicate.",
+ "minLength": 1,
+ "examples": [
+ "Permission denied, scanning aborted"
+ ]
+ }
+ }
+ }
+ },
+ "options": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "A configuration option used for this scan.",
+ "required": [
+ "name",
+ "value"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The configuration option name.",
+ "maxLength": 255,
+ "minLength": 1,
+ "examples": [
+ "DAST_FF_ENABLE_BAS",
+ "DOCKER_TLS_CERTDIR",
+ "DS_MAX_DEPTH",
+ "SECURE_LOG_LEVEL"
+ ]
+ },
+ "source": {
+ "type": "string",
+ "description": "The source of this option.",
+ "enum": [
+ "argument",
+ "file",
+ "env_variable",
+ "other"
+ ]
+ },
+ "value": {
+ "type": [
+ "boolean",
+ "integer",
+ "null",
+ "string"
+ ],
+ "description": "The value used for this scan.",
+ "examples": [
+ true,
+ 2,
+ null,
+ "fatal",
+ ""
+ ]
+ }
+ }
+ }
+ },
+ "analyzer": {
+ "type": "object",
+ "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "gitlab-dast"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the analyzer, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "GitLab DAST"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "pattern": "^https?://.+",
+ "description": "A link to more information about the analyzer.",
+ "examples": [
+ "https://docs.gitlab.com/ee/user/application_security/dast"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the analyzer.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the analyzer.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ }
+ }
+ },
+ "scanner": {
+ "type": "object",
+ "description": "Object defining the scanner used to perform the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the scanner.",
+ "minLength": 1,
+ "examples": [
+ "my-sast-scanner"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the scanner, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "My SAST Scanner"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "A link to more information about the scanner.",
+ "examples": [
+ "https://scanner.url"
+ ]
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the scanner.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ },
+ "vendor": {
+ "description": "The vendor/maintainer of the scanner.",
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ }
+ }
+ },
+ "start_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$",
+ "examples": [
+ "2020-02-14T16:01:59"
+ ]
+ },
+ "status": {
+ "type": "string",
+ "description": "Result of the scan.",
+ "enum": [
+ "success",
+ "failure"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "description": "Type of the scan.",
+ "enum": [
+ "secret_detection"
+ ]
+ },
+ "primary_identifiers": {
+ "type": "array",
+ "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ }
+ }
+ },
+ "schema": {
+ "type": "string",
+ "description": "URI pointing to the validating security report schema.",
+ "pattern": "^https?://.+"
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the schema to which the JSON report conforms.",
+ "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+ },
+ "vulnerabilities": {
+ "type": "array",
+ "description": "Array of vulnerability objects.",
+ "items": {
+ "type": "object",
+ "description": "Describes the vulnerability using GitLab Flavored Markdown",
+ "required": [
+ "id",
+ "identifiers",
+ "location"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "maxLength": 255,
+ "description": "The name of the vulnerability. This must not include the finding's specific information."
+ },
+ "description": {
+ "type": "string",
+ "maxLength": 1048576,
+ "description": "A long text section describing the vulnerability more fully."
+ },
+ "severity": {
+ "type": "string",
+ "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+ "enum": [
+ "Info",
+ "Unknown",
+ "Low",
+ "Medium",
+ "High",
+ "Critical"
+ ]
+ },
+ "solution": {
+ "type": "string",
+ "maxLength": 7000,
+ "description": "Explanation of how to fix the vulnerability."
+ },
+ "identifiers": {
+ "type": "array",
+ "minItems": 1,
+ "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "pattern": "^(https?|ftp)://.+"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ },
+ "cvss_vectors": {
+ "type": "array",
+ "minItems": 1,
+ "maxItems": 10,
+ "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.",
+ "items": {
+ "oneOf": [
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 16,
+ "maxLength": 128,
+ "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ },
+ {
+ "type": "object",
+ "properties": {
+ "vendor": {
+ "type": "string",
+ "minLength": 1,
+ "default": "unknown"
+ },
+ "vector": {
+ "type": "string",
+ "minLength": 32,
+ "maxLength": 128,
+ "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$"
+ }
+ },
+ "required": [
+ "vendor",
+ "vector"
+ ]
+ }
+ ]
+ }
+ },
+ "links": {
+ "type": "array",
+ "description": "An array of references to external documentation or articles that describe the vulnerability.",
+ "items": {
+ "type": "object",
+ "required": [
+ "url"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the vulnerability details link."
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the vulnerability details document.",
+ "pattern": "^(https?|ftp)://.+"
+ }
+ }
+ }
+ },
+ "details": {
+ "$ref": "#/definitions/named_list/properties/items"
+ },
+ "tracking": {
+ "type": "object",
+ "description": "Describes how this vulnerability should be tracked as the project changes.",
+ "oneOf": [
+ {
+ "description": "Declares that a series of items should be tracked using source-specific tracking methods.",
+ "required": [
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "source"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "description": "An item that should be tracked using source-specific tracking methods.",
+ "type": "object",
+ "required": [
+ "signatures"
+ ],
+ "properties": {
+ "file": {
+ "type": "string",
+ "description": "Path to the file where the vulnerability is located."
+ },
+ "start_line": {
+ "type": "number",
+ "description": "The first line of the file that includes the vulnerability."
+ },
+ "end_line": {
+ "type": "number",
+ "description": "The last line of the file that includes the vulnerability."
+ },
+ "signatures": {
+ "type": "array",
+ "description": "An array of calculated tracking signatures for this tracking item.",
+ "minItems": 1,
+ "items": {
+ "description": "A calculated tracking signature value and metadata.",
+ "type": "object",
+ "required": [
+ "algorithm",
+ "value"
+ ],
+ "properties": {
+ "algorithm": {
+ "type": "string",
+ "description": "The algorithm used to generate the signature."
+ },
+ "value": {
+ "type": "string",
+ "description": "The result of this signature algorithm."
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "Each tracking type must declare its own type."
+ }
+ }
+ },
+ "flags": {
+ "description": "Flags that can be attached to vulnerabilities.",
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Informational flags identified and assigned to a vulnerability.",
+ "required": [
+ "type",
+ "origin",
+ "description"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Result of the scan.",
+ "enum": [
+ "flagged-as-likely-false-positive"
+ ]
+ },
+ "origin": {
+ "minLength": 1,
+ "description": "Tool that issued the flag.",
+ "type": "string"
+ },
+ "description": {
+ "minLength": 1,
+ "description": "What the flag is about.",
+ "type": "string"
+ }
+ }
+ }
+ },
+ "location": {
+ "required": [
+ "commit"
+ ],
+ "type": "object",
+ "properties": {
+ "file": {
+ "type": "string",
+ "description": "Path to the file where the vulnerability is located"
+ },
+ "commit": {
+ "type": "object",
+ "description": "Represents the commit in which the vulnerability was detected",
+ "required": [
+ "sha"
+ ],
+ "properties": {
+ "author": {
+ "type": "string"
+ },
+ "date": {
+ "type": "string"
+ },
+ "message": {
+ "type": "string"
+ },
+ "sha": {
+ "type": "string",
+ "minLength": 1
+ }
+ }
+ },
+ "start_line": {
+ "type": "number",
+ "description": "The first line of the code affected by the vulnerability"
+ },
+ "end_line": {
+ "type": "number",
+ "description": "The last line of the code affected by the vulnerability"
+ },
+ "class": {
+ "type": "string",
+ "description": "Provides the name of the class where the vulnerability is located"
+ },
+ "method": {
+ "type": "string",
+ "description": "Provides the name of the method where the vulnerability is located"
+ }
+ }
+ },
+ "raw_source_code_extract": {
+ "type": "string",
+ "description": "Provides an unsanitized excerpt of the affected source code."
+ }
+ }
+ }
+ },
+ "remediations": {
+ "type": "array",
+ "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+ "items": {
+ "type": "object",
+ "required": [
+ "fixes",
+ "summary",
+ "diff"
+ ],
+ "properties": {
+ "fixes": {
+ "type": "array",
+ "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+ "items": {
+ "type": "object",
+ "required": [
+ "id"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ }
+ }
+ }
+ },
+ "summary": {
+ "type": "string",
+ "minLength": 1,
+ "description": "An overview of how the vulnerabilities were fixed."
+ },
+ "diff": {
+ "type": "string",
+ "minLength": 1,
+ "description": "A base64-encoded remediation code diff, compatible with git apply."
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb
index 43fb5cdbbe6..b8c8cfa802c 100644
--- a/lib/gitlab/ci/status/build/cancelable.rb
+++ b/lib/gitlab/ci/status/build/cancelable.rb
@@ -6,7 +6,7 @@ module Gitlab
module Build
class Cancelable < Status::Extended
def has_action?
- can?(user, :update_build, subject)
+ can?(user, :cancel_build, subject)
end
def action_icon
diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb
index 1ba78b357e5..fe4f6db9549 100644
--- a/lib/gitlab/ci/status/composite.rb
+++ b/lib/gitlab/ci/status/composite.rb
@@ -61,6 +61,8 @@ module Gitlab
'running'
elsif any_of?(:waiting_for_resource)
'waiting_for_resource'
+ elsif any_of?(:waiting_for_callback)
+ 'waiting_for_callback'
elsif any_of?(:manual)
'manual'
elsif any_of?(:scheduled)
diff --git a/lib/gitlab/ci/status/waiting_for_callback.rb b/lib/gitlab/ci/status/waiting_for_callback.rb
new file mode 100644
index 00000000000..0184a910ede
--- /dev/null
+++ b/lib/gitlab/ci/status/waiting_for_callback.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Status
+ class WaitingForCallback < Status::Core
+ def text
+ s_('CiStatusText|Waiting')
+ end
+
+ def label
+ s_('CiStatusLabel|waiting for callback')
+ end
+
+ def icon
+ 'status_pending'
+ end
+
+ def favicon
+ 'favicon_status_pending'
+ end
+
+ def group
+ 'waiting-for-callback'
+ end
+
+ def details_path
+ nil
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml b/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml
index 356062c734e..324128678de 100644
--- a/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml
@@ -12,9 +12,9 @@ include:
docker-build:
variables:
- COSIGN_YES: "true" # Used by Cosign to skip confirmation prompts for non-destructive operations
+ COSIGN_YES: "true" # Used by Cosign to skip confirmation prompts for non-destructive operations
id_tokens:
- SIGSTORE_ID_TOKEN: # Used by Cosign to get certificate from Fulcio
+ SIGSTORE_ID_TOKEN: # Used by Cosign to get certificate from Fulcio
aud: sigstore
after_script:
- apk add --update cosign
diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
index 2d04c97b32e..6898923bc53 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- AUTO_BUILD_IMAGE_VERSION: 'v1.44.0'
+ AUTO_BUILD_IMAGE_VERSION: 'v1.49.0'
build:
stage: build
diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
index 2d04c97b32e..6898923bc53 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- AUTO_BUILD_IMAGE_VERSION: 'v1.44.0'
+ AUTO_BUILD_IMAGE_VERSION: 'v1.49.0'
build:
stage: build
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index 4d53b92763a..7d923245d79 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1'
+ DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0'
.dast-auto-deploy:
image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}"
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 390824e8e49..0f8d5bf6d8f 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1'
+ AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0'
.auto-deploy:
image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}"
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
index a9681c0f927..e29d18ea45a 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml
@@ -1,5 +1,5 @@
variables:
- AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1'
+ AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0'
.auto-deploy:
image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}"
diff --git a/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml
index d7a6104082d..4c89497fa97 100644
--- a/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml
@@ -46,13 +46,30 @@ kaniko-build:
# Write credentials to access Gitlab Container Registry within the runner/ci
- echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json
# Build and push the container. To disable push add --no-push
- - DOCKERFILE_PATH=${DOCKERFILE_PATH:-"$KANIKO_BUILD_CONTEXT/Dockerfile"}
+ # Both Dockerfile and Containerfile are supported. For retrocompatibility, if both files are present, Dockerfile will be used.
+ - |
+ if [ -z "$DOCKERFILE_PATH" ]; then
+ if [ -f "$KANIKO_BUILD_CONTEXT/Dockerfile" ]; then
+ DOCKERFILE_PATH="$KANIKO_BUILD_CONTEXT/Dockerfile"
+ elif [ -n "$CONTAINERFILE_PATH" ]; then
+ DOCKERFILE_PATH="$CONTAINERFILE_PATH"
+ elif [ -f "$KANIKO_BUILD_CONTEXT/Containerfile" ]; then
+ DOCKERFILE_PATH="$KANIKO_BUILD_CONTEXT/Containerfile"
+ else \
+ echo "No suitable configuration for the build context have been found. Please check your configuration."
+ exit 1
+ fi
+ fi
+ - echo $DOCKERFILE_PATH
- /kaniko/executor --context $KANIKO_BUILD_CONTEXT --dockerfile $DOCKERFILE_PATH --destination $IMAGE_TAG $KANIKO_ARGS
- # Run this job in a branch/tag where a Dockerfile exists
+ # Run this job in a branch/tag where a Containerfile/Dockerfile exists
rules:
- exists:
+ - Containerfile
- Dockerfile
- # custom Dockerfile path
+ # custom Containerfile/Dockerfile path
+ # If both variables are set, DOCKERFILE_PATH will be used
- if: $DOCKERFILE_PATH
+ - if: $CONTAINERFILE_PATH
# custom build context without an explicit Dockerfile path
- if: $KANIKO_BUILD_CONTEXT != $CI_PROJECT_DIR
diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
index d2b929cf995..0ba4f9715c5 100644
--- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
@@ -50,7 +50,11 @@ variables:
- gitlab-terraform plan-json
resource_group: ${TF_STATE_NAME}
artifacts:
- # The next line, which disables public access to pipeline artifacts, may not be available everywhere.
+ # Terraform's cache files can include secrets which can be accidentally exposed.
+ # Please exercise caution when utilizing secrets in your Terraform infrastructure and
+ # consider limiting access to artifacts or take other security measures to protect sensitive information.
+ #
+ # The next line, which disables public access to pipeline artifacts, is not available on GitLab.com.
# See: https://docs.gitlab.com/ee/ci/yaml/#artifactspublic
public: false
paths:
diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
index c1a90955f7f..8c9e0a329dd 100644
--- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml
@@ -19,6 +19,7 @@ browser_performance:
SITESPEED_IMAGE: sitespeedio/sitespeed.io
SITESPEED_VERSION: 26.1.0
SITESPEED_OPTIONS: ''
+ SITESPEED_DOCKER_OPTIONS: ''
services:
- docker:dind
script:
@@ -48,7 +49,7 @@ browser_performance:
HTTP_PROXY \
NO_PROXY \
) \
- --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS
+ $SITESPEED_DOCKER_OPTIONS --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS
- mv sitespeed-results/data/performance.json browser-performance.json
artifacts:
paths:
diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml
index adc92fde5ae..3f4c0c53850 100644
--- a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml
@@ -19,6 +19,7 @@ browser_performance:
SITESPEED_IMAGE: sitespeedio/sitespeed.io
SITESPEED_VERSION: latest
SITESPEED_OPTIONS: ''
+ SITESPEED_DOCKER_OPTIONS: ''
services:
- docker:dind
script:
@@ -48,7 +49,7 @@ browser_performance:
HTTP_PROXY \
NO_PROXY \
) \
- --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS
+ $SITESPEED_DOCKER_OPTIONS --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS
- mv sitespeed-results/data/performance.json browser-performance.json
artifacts:
paths:
diff --git a/lib/gitlab/ci/yaml_processor/dag.rb b/lib/gitlab/ci/yaml_processor/dag.rb
index 4a122c73e80..d3047385c99 100644
--- a/lib/gitlab/ci/yaml_processor/dag.rb
+++ b/lib/gitlab/ci/yaml_processor/dag.rb
@@ -17,13 +17,15 @@ module Gitlab
def self.check_circular_dependencies!(jobs)
new(jobs).tsort
- rescue TSort::Cyclic
- raise ValidationError, 'The pipeline has circular dependencies'
+ rescue TSort::Cyclic => e
+ raise ValidationError, "The pipeline has circular dependencies: #{e.message}"
end
def tsort_each_child(node, &block)
return unless @nodes[node]
+ raise TSort::Cyclic, "self-dependency: #{node}" if @nodes[node].include?(node)
+
@nodes[node].each(&block)
end
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index 6207b595fc6..2435d128bf2 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -8,8 +8,7 @@ module Gitlab
class Result
attr_reader :errors, :warnings,
:root_variables, :root_variables_with_prefill_data,
- :stages, :jobs,
- :workflow_rules, :workflow_name
+ :stages, :jobs, :workflow_rules, :workflow_name
def initialize(ci_config: nil, errors: [], warnings: [])
@ci_config = ci_config
@@ -124,7 +123,8 @@ module Gitlab
trigger: job[:trigger],
bridge_needs: job.dig(:needs, :bridge)&.first,
release: job[:release],
- publish: job[:publish]
+ publish: job[:publish],
+ pages: job[:pages]
}.compact }.compact
end
diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb
index bfa3112b795..0834fda9cf9 100644
--- a/lib/gitlab/composer/version_index.rb
+++ b/lib/gitlab/composer/version_index.rb
@@ -48,8 +48,7 @@ module Gitlab
end
def package_source(package)
- use_http_url = package.project.public? || Feature.disabled?(:composer_use_ssh_source_urls, package.project)
- git_url = use_http_url ? package.project.http_url_to_repo : package.project.ssh_url_to_repo
+ git_url = package.project.public? ? package.project.http_url_to_repo : package.project.ssh_url_to_repo
{
'type' => 'git',
diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb
index 87b7cab3f6d..c7dd11b0432 100644
--- a/lib/gitlab/config/entry/validators.rb
+++ b/lib/gitlab/config/entry/validators.rb
@@ -56,8 +56,7 @@ module Gitlab
mutually_exclusive_keys = value.try(:keys).to_a & options[:in]
if mutually_exclusive_keys.length > 1
- record.errors.add(attribute, "please use only one of the following keys: " +
- mutually_exclusive_keys.join(', '))
+ record.errors.add(attribute, "these keys cannot be used together: #{mutually_exclusive_keys.join(', ')}")
end
end
end
diff --git a/lib/gitlab/config_checker/puma_rugged_checker.rb b/lib/gitlab/config_checker/puma_rugged_checker.rb
deleted file mode 100644
index 82c59f3328b..00000000000
--- a/lib/gitlab/config_checker/puma_rugged_checker.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ConfigChecker
- module PumaRuggedChecker
- extend self
- extend Gitlab::Git::RuggedImpl::UseRugged
-
- def check
- notices = []
-
- if running_puma_with_multiple_threads? && rugged_enabled_through_feature_flag?
- link_start = '<a href="https://docs.gitlab.com/ee/administration/operations/puma.html#performance-caveat-when-using-puma-with-rugged">'
- link_end = '</a>'
- notices << {
- type: 'warning',
- message: _('Puma is running with a thread count above 1 and the Rugged '\
- 'service is enabled. This may decrease performance in some environments. '\
- 'See our %{link_start}documentation%{link_end} '\
- 'for details of this issue.') % { link_start: link_start, link_end: link_end }
- }
- end
-
- notices
- end
- end
- end
-end
diff --git a/lib/gitlab/database/dictionary.rb b/lib/gitlab/database/dictionary.rb
new file mode 100644
index 00000000000..7b0c8560a26
--- /dev/null
+++ b/lib/gitlab/database/dictionary.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ class Dictionary
+ def initialize(file_path)
+ @file_path = file_path
+ @data = YAML.load_file(file_path)
+ end
+
+ def name_and_schema
+ [key_name, gitlab_schema.to_sym]
+ end
+
+ def table_name
+ data['table_name']
+ end
+
+ def view_name
+ data['view_name']
+ end
+
+ def milestone
+ data['milestone']
+ end
+
+ def gitlab_schema
+ data['gitlab_schema']
+ end
+
+ def schema?(schema_name)
+ gitlab_schema == schema_name.to_s
+ end
+
+ def key_name
+ table_name || view_name
+ end
+
+ def validate!
+ return true unless gitlab_schema.nil?
+
+ raise(
+ GitlabSchema::UnknownSchemaError,
+ "#{file_path} must specify a valid gitlab_schema for #{key_name}. " \
+ "See #{help_page_url}"
+ )
+ end
+
+ private
+
+ attr_reader :file_path, :data
+
+ def help_page_url
+ # rubocop:disable Gitlab/DocUrl -- link directly to docs.gitlab.com, always
+ 'https://docs.gitlab.com/ee/development/database/database_dictionary.html'
+ # rubocop:enable Gitlab/DocUrl
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/dynamic_model_helpers.rb b/lib/gitlab/database/dynamic_model_helpers.rb
index 83edf77f37e..18854530278 100644
--- a/lib/gitlab/database/dynamic_model_helpers.rb
+++ b/lib/gitlab/database/dynamic_model_helpers.rb
@@ -5,7 +5,7 @@ module Gitlab
module DynamicModelHelpers
BATCH_SIZE = 1_000
- def define_batchable_model(table_name, connection:)
+ def define_batchable_model(table_name, connection:, primary_key: nil)
klass = Class.new(ActiveRecord::Base) do
include EachBatch
@@ -13,6 +13,7 @@ module Gitlab
self.inheritance_column = :_type_disabled
end
+ klass.primary_key = primary_key if connection.primary_keys(table_name).length > 1
klass.connection = connection
klass
end
diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb
index 31ceb898eee..ecb45622061 100644
--- a/lib/gitlab/database/gitlab_schema.rb
+++ b/lib/gitlab/database/gitlab_schema.rb
@@ -31,6 +31,7 @@ module Gitlab
'_test_gitlab_main_cell_' => :gitlab_main_cell,
'_test_gitlab_main_' => :gitlab_main,
'_test_gitlab_ci_' => :gitlab_ci,
+ '_test_gitlab_jh_' => :gitlab_jh,
'_test_gitlab_embedding_' => :gitlab_embedding,
'_test_gitlab_geo_' => :gitlab_geo,
'_test_gitlab_pm_' => :gitlab_pm,
@@ -138,19 +139,19 @@ module Gitlab
end
def self.deleted_tables_to_schema
- @deleted_tables_to_schema ||= self.build_dictionary('deleted_tables').to_h
+ @deleted_tables_to_schema ||= self.build_dictionary('deleted_tables').map(&:name_and_schema).to_h
end
def self.deleted_views_to_schema
- @deleted_views_to_schema ||= self.build_dictionary('deleted_views').to_h
+ @deleted_views_to_schema ||= self.build_dictionary('deleted_views').map(&:name_and_schema).to_h
end
def self.tables_to_schema
- @tables_to_schema ||= self.build_dictionary('').to_h
+ @tables_to_schema ||= self.build_dictionary('').map(&:name_and_schema).to_h
end
def self.views_to_schema
- @views_to_schema ||= self.build_dictionary('views').to_h
+ @views_to_schema ||= self.build_dictionary('views').map(&:name_and_schema).to_h
end
def self.schema_names
@@ -159,21 +160,9 @@ module Gitlab
def self.build_dictionary(scope)
Dir.glob(dictionary_path_globs(scope)).map do |file_path|
- data = YAML.load_file(file_path)
-
- key_name = data['table_name'] || data['view_name']
-
- # rubocop:disable Gitlab/DocUrl
- if data['gitlab_schema'].nil?
- raise(
- UnknownSchemaError,
- "#{file_path} must specify a valid gitlab_schema for #{key_name}. " \
- "See https://docs.gitlab.com/ee/development/database/database_dictionary.html"
- )
- end
- # rubocop:enable Gitlab/DocUrl
-
- [key_name, data['gitlab_schema'].to_sym]
+ dictionary = Dictionary.new(file_path)
+ dictionary.validate!
+ dictionary
end
end
end
diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb
index 41044816de9..1d17c2ca608 100644
--- a/lib/gitlab/database/migration.rb
+++ b/lib/gitlab/database/migration.rb
@@ -56,6 +56,10 @@ module Gitlab
include Gitlab::Database::Migrations::RunnerBackoff::MigrationHelpers
end
+ class V2_2 < V2_1
+ include Gitlab::Database::Migrations::MilestoneMixin
+ end
+
def self.[](version)
version = version.to_s
name = "V#{version.tr('.', '_')}"
@@ -66,7 +70,7 @@ module Gitlab
# The current version to be used in new migrations
def self.current_version
- 2.1
+ 2.2
end
end
end
diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb
index efcceafda90..a57bce789c7 100644
--- a/lib/gitlab/database/migration_helpers.rb
+++ b/lib/gitlab/database/migration_helpers.rb
@@ -18,8 +18,8 @@ module Gitlab
include AsyncConstraints::MigrationHelpers
include WraparoundVacuumHelpers
- def define_batchable_model(table_name, connection: self.connection)
- super(table_name, connection: connection)
+ def define_batchable_model(table_name, connection: self.connection, primary_key: nil)
+ super(table_name, connection: connection, primary_key: primary_key)
end
def each_batch(table_name, connection: self.connection, **kwargs)
@@ -821,6 +821,7 @@ module Gitlab
primary_key: :id,
batch_size: 20_000,
sub_batch_size: 1000,
+ pause_ms: 100,
interval: 2.minutes
)
@@ -848,6 +849,7 @@ module Gitlab
conversions.keys,
conversions.values,
job_interval: interval,
+ pause_ms: pause_ms,
batch_size: batch_size,
sub_batch_size: sub_batch_size)
end
diff --git a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb
index 11f1e62e8b9..d1edb739b85 100644
--- a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb
+++ b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb
@@ -4,12 +4,16 @@ module Gitlab
module Database
module MigrationHelpers
module ConvertToBigint
- # This helper is extracted for the purpose of
- # https://gitlab.com/gitlab-org/gitlab/-/issues/392815
- # so that we can test all combinations just once,
- # and simplify migration tests.
- #
- # Once we are done with the PK conversions we can remove this.
+ INDEX_OPTIONS_MAP = {
+ unique: :unique,
+ order: :orders,
+ opclass: :opclasses,
+ where: :where,
+ type: :type,
+ using: :using,
+ comment: :comment
+ }.freeze
+
def com_or_dev_or_test_but_not_jh?
return true if Gitlab.dev_or_test_env?
@@ -29,6 +33,78 @@ module Gitlab
column.sql_type == 'bigint' && temp_column.sql_type == 'integer'
end
+
+ def add_bigint_column_indexes(table_name, int_column_name)
+ bigint_column_name = convert_to_bigint_column(int_column_name)
+
+ unless column_exists?(table_name.to_s, bigint_column_name)
+ raise "Bigint column '#{bigint_column_name}' does not exist on #{table_name}"
+ end
+
+ indexes(table_name).each do |i|
+ next unless Array(i.columns).join(' ').match?(/\b#{int_column_name}\b/)
+
+ create_bigint_index(table_name, i, int_column_name, bigint_column_name)
+ end
+ end
+
+ # default 'index_name' method is not used because this method can be reused while swapping/dropping the indexes
+ def bigint_index_name(int_column_index_name)
+ # First 20 digits of the hash is chosen to make sure it fits the 63 chars limit
+ digest = Digest::SHA256.hexdigest(int_column_index_name).first(20)
+ "bigint_idx_#{digest}"
+ end
+
+ private
+
+ def create_bigint_index(table_name, index_definition, int_column_name, bigint_column_name)
+ index_attributes = index_definition.as_json
+ index_options = INDEX_OPTIONS_MAP
+ .transform_values { |key| index_attributes[key.to_s] }
+ .select { |_, v| v.present? }
+
+ bigint_index_options = create_bigint_options(
+ index_options,
+ index_definition.name,
+ int_column_name,
+ bigint_column_name
+ )
+
+ add_concurrent_index(
+ table_name,
+ bigint_index_columns(int_column_name, bigint_column_name, index_definition.columns),
+ name: bigint_index_options.delete(:name),
+ ** bigint_index_options
+ )
+ end
+
+ def bigint_index_columns(int_column_name, bigint_column_name, int_index_columns)
+ if int_index_columns.is_a?(String)
+ int_index_columns.gsub(/\b#{int_column_name}\b/, bigint_column_name)
+ else
+ int_index_columns.map do |column|
+ column == int_column_name.to_s ? bigint_column_name : column
+ end
+ end
+ end
+
+ def create_bigint_options(index_options, int_index_name, int_column_name, bigint_column_name)
+ index_options[:name] = bigint_index_name(int_index_name)
+ index_options[:where]&.gsub!(/\b#{int_column_name}\b/, bigint_column_name)
+
+ # ordering on multiple columns will return a Hash instead of string
+ index_options[:order] =
+ if index_options[:order].is_a?(Hash)
+ index_options[:order].to_h do |column, order|
+ column = bigint_column_name if column == int_column_name
+ [column, order]
+ end
+ else
+ index_options[:order]&.gsub(/\b#{int_column_name}\b/, bigint_column_name)
+ end
+
+ index_options.select { |_, v| v.present? }
+ end
end
end
end
diff --git a/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb b/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb
index 555efb58606..7f215bc0db7 100644
--- a/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb
+++ b/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb
@@ -13,7 +13,7 @@ module Gitlab
# 3. Introduce the migration again for self-managed.
#
def can_execute_on?(*tables)
- return false unless Gitlab.com? || Gitlab.dev_or_test_env?
+ return false unless Gitlab.com_except_jh? || Gitlab.dev_or_test_env?
if wraparound_prevention_on_tables?(tables)
Gitlab::AppLogger.info(message: "Wraparound prevention vacuum detected", class: self.class)
diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
index 64cde273a59..3d4ac113bf6 100644
--- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
+++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb
@@ -72,6 +72,7 @@ module Gitlab
batch_max_value: nil,
batch_class_name: BATCH_CLASS_NAME,
batch_size: BATCH_SIZE,
+ pause_ms: 100,
max_batch_size: nil,
sub_batch_size: SUB_BATCH_SIZE,
gitlab_schema: nil
@@ -105,6 +106,7 @@ module Gitlab
column_name: batch_column_name,
job_arguments: job_arguments,
interval: job_interval,
+ pause_ms: pause_ms,
min_value: batch_min_value,
max_value: batch_max_value,
batch_class_name: batch_class_name,
diff --git a/lib/gitlab/database/migrations/milestone_mixin.rb b/lib/gitlab/database/migrations/milestone_mixin.rb
index 10bc0c192e7..7d78f74d237 100644
--- a/lib/gitlab/database/migrations/milestone_mixin.rb
+++ b/lib/gitlab/database/migrations/milestone_mixin.rb
@@ -19,11 +19,10 @@ module Gitlab
end
end
- def initialize(name = class_name, version = nil, type = nil)
- raise MilestoneNotSetError, "Milestone is not set for #{self.class.name}" if milestone.nil?
+ def initialize(name = self.class.name, version = nil, _type = nil)
+ raise MilestoneNotSetError, "Milestone is not set for #{name}" if milestone.nil?
super(name, version)
- @version = Gitlab::Database::Migrations::Version.new(version, milestone, type)
end
def milestone # rubocop:disable Lint/DuplicateMethods
diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb
index bb70d052e3e..83cd446534c 100644
--- a/lib/gitlab/database/partitioning/partition_manager.rb
+++ b/lib/gitlab/database/partitioning/partition_manager.rb
@@ -89,6 +89,8 @@ module Gitlab
Gitlab::AppLogger.info(message: "Created partition",
partition_name: partition.partition_name,
table_name: partition.table)
+
+ lock_partitions_for_writes(partition) if should_lock_for_writes?
end
model.partitioning_strategy.after_adding_partitions
@@ -205,6 +207,23 @@ module Gitlab
end
end
end
+
+ def should_lock_for_writes?
+ Feature.enabled?(:automatic_lock_writes_on_partition_tables, type: :ops) &&
+ Gitlab::Database.database_mode == Gitlab::Database::MODE_MULTIPLE_DATABASES &&
+ connection != model.connection
+ end
+ strong_memoize_attr :should_lock_for_writes?
+
+ def lock_partitions_for_writes(partition)
+ table_name = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{partition.partition_name}"
+ Gitlab::Database::LockWritesManager.new(
+ table_name: table_name,
+ connection: connection,
+ database_name: @connection_name,
+ with_retries: !connection.transaction_open?
+ ).lock_writes
+ end
end
end
end
diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
index 1ce0a44e37f..b486ddb8e76 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
@@ -8,7 +8,8 @@ module Gitlab
include ::Gitlab::Database::MigrationHelpers
include ::Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers
- ALLOWED_TABLES = %w[audit_events web_hook_logs].freeze
+ ALLOWED_TABLES = %w[audit_events web_hook_logs merge_request_diff_files merge_request_diff_commits].freeze
+
ERROR_SCOPE = 'table partitioning'
MIGRATION_CLASS_NAME = "::#{module_parent_name}::BackfillPartitionedTable"
@@ -16,6 +17,60 @@ module Gitlab
BATCH_INTERVAL = 2.minutes.freeze
BATCH_SIZE = 50_000
SUB_BATCH_SIZE = 2_500
+ PARTITION_BUFFER = 6
+ MIN_ID = 1
+
+ # Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a int/bigint column.
+ # One partition is created per partition_size between 1 and MAX(column_name). Also installs a trigger on
+ # the original table to copy writes into the partitioned table. To copy over historic data from before creation
+ # of the partitioned table, use the `enqueue_partitioning_data_migration` helper in a post-deploy migration.
+ # Note: If the original table is empty the system creates 6 partitions in the new table.
+ #
+ # A copy of the original table is required as PG currently does not support partitioning existing tables.
+ #
+ # Example:
+ #
+ # partition_table_by_int_range :merge_request_diff_commits, :merge_request_diff_id, partition_size: 500, primary_key: ['merge_request_diff_id', 'relative_order']
+ #
+ # Options are:
+ # :partition_size - a int specifying the partition size
+ # :primary_key - a array specifying the primary query of the new table
+ #
+ # Note: The system always adds a buffer of 6 partitions.
+ def partition_table_by_int_range(table_name, column_name, partition_size:, primary_key:)
+ Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode!
+
+ assert_table_is_allowed(table_name)
+
+ assert_not_in_transaction_block(scope: ERROR_SCOPE)
+
+ current_primary_key = Array.wrap(connection.primary_key(table_name))
+ raise "primary key not defined for #{table_name}" if current_primary_key.blank?
+
+ partition_column = find_column_definition(table_name, column_name)
+ raise "partition column #{column_name} does not exist on #{table_name}" if partition_column.nil?
+
+ primary_key = Array.wrap(primary_key).map(&:to_s)
+ raise "the partition column must be part of the primary key" unless primary_key.include?(column_name.to_s)
+
+ primary_key_objects = connection.columns(table_name).select { |column| primary_key.include?(column.name) }
+
+ raise 'partition_size must be greater than 1' unless partition_size > 1
+
+ max_id = Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do
+ Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection.with_suppressed do
+ define_batchable_model(table_name, connection: connection).maximum(column_name) || partition_size * PARTITION_BUFFER
+ end
+ end
+
+ partitioned_table_name = make_partitioned_table_name(table_name)
+
+ with_lock_retries do
+ create_range_id_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key_objects)
+ create_int_range_partitions(partitioned_table_name, partition_size, MIN_ID, max_id)
+ create_trigger_to_sync_tables(table_name, partitioned_table_name, current_primary_key)
+ end
+ end
# Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column.
# One partition is created per month between the given `min_date` and `max_date`. Also installs a trigger on
@@ -332,6 +387,34 @@ module Gitlab
connection.columns(table).find { |c| c.name == column.to_s }
end
+ def create_range_id_partitioned_copy(source_table_name, partitioned_table_name, partition_column, primary_keys)
+ if table_exists?(partitioned_table_name)
+ Gitlab::AppLogger.warn "Partitioned table not created because it already exists" \
+ " (this may be due to an aborted migration or similar): table_name: #{partitioned_table_name} "
+ return
+ end
+
+ tmp_partitioning_column_name = "#{partition_column.name}_tmp"
+
+ temporary_columns = primary_keys.map { |key| "#{key.name}_tmp" }.join(", ")
+ temporary_columns_statement = build_temporary_columns_statement(primary_keys)
+
+ transaction do
+ execute(<<~SQL)
+ CREATE TABLE #{partitioned_table_name} (
+ LIKE #{source_table_name} INCLUDING ALL EXCLUDING INDEXES,
+ #{temporary_columns_statement},
+ PRIMARY KEY (#{temporary_columns})
+ ) PARTITION BY RANGE (#{tmp_partitioning_column_name})
+ SQL
+
+ primary_keys.each do |key|
+ remove_column(partitioned_table_name, key.name)
+ rename_column(partitioned_table_name, "#{key.name}_tmp", key.name)
+ end
+ end
+ end
+
def create_range_partitioned_copy(source_table_name, partitioned_table_name, partition_column, primary_key)
if table_exists?(partitioned_table_name)
Gitlab::AppLogger.warn "Partitioned table not created because it already exists" \
@@ -382,6 +465,20 @@ module Gitlab
end
end
+ def create_int_range_partitions(table_name, partition_size, min_id, max_id)
+ lower_bound = min_id
+ upper_bound = min_id + partition_size
+
+ end_id = max_id + PARTITION_BUFFER * partition_size # Adds a buffer of 6 partitions
+
+ while lower_bound < end_id
+ create_range_partition_safely("#{table_name}_#{lower_bound}", table_name, lower_bound, upper_bound)
+
+ lower_bound += partition_size
+ upper_bound += partition_size
+ end
+ end
+
def to_sql_date_literal(date)
connection.quote(date.strftime('%Y-%m-%d'))
end
@@ -411,19 +508,23 @@ module Gitlab
return
end
+ unique_key = Array.wrap(unique_key)
+
delimiter = ",\n "
column_names = connection.columns(partitioned_table_name).map(&:name)
set_statements = build_set_statements(column_names, unique_key)
insert_values = column_names.map { |name| "NEW.#{name}" }
+ delete_where_statement = unique_key.map { |unique_key| "#{unique_key} = OLD.#{unique_key}" }.join(' AND ')
+ update_where_statement = unique_key.map { |unique_key| "#{partitioned_table_name}.#{unique_key} = NEW.#{unique_key}" }.join(' AND ')
create_trigger_function(name, replace: false) do
<<~SQL
IF (TG_OP = 'DELETE') THEN
- DELETE FROM #{partitioned_table_name} where #{unique_key} = OLD.#{unique_key};
+ DELETE FROM #{partitioned_table_name} where #{delete_where_statement};
ELSIF (TG_OP = 'UPDATE') THEN
UPDATE #{partitioned_table_name}
SET #{set_statements.join(delimiter)}
- WHERE #{partitioned_table_name}.#{unique_key} = NEW.#{unique_key};
+ WHERE #{update_where_statement};
ELSIF (TG_OP = 'INSERT') THEN
INSERT INTO #{partitioned_table_name} (#{column_names.join(delimiter)})
VALUES (#{insert_values.join(delimiter)});
@@ -433,8 +534,16 @@ module Gitlab
end
end
+ def build_temporary_columns_statement(columns)
+ columns.map do |column|
+ type = column.name == 'id' || column.name.end_with?('_id') ? 'bigint' : column.sql_type
+
+ "#{column.name}_tmp #{type} NOT NULL"
+ end.join(", ")
+ end
+
def build_set_statements(column_names, unique_key)
- column_names.reject { |name| name == unique_key }.map { |name| "#{name} = NEW.#{name}" }
+ column_names.reject { |name| unique_key.include?(name) }.map { |name| "#{name} = NEW.#{name}" }
end
def create_sync_trigger(table_name, trigger_name, function_name)
diff --git a/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb b/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb
index eb55ebc7619..c2f94b7b0e6 100644
--- a/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb
+++ b/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb
@@ -8,7 +8,7 @@ module Gitlab
class PartitioningRoutingAnalyzer < Database::QueryAnalyzers::Base
RoutingTableNotUsedError = Class.new(QueryAnalyzerError)
- ENABLED_TABLES = %w[ci_builds_metadata].freeze
+ ENABLED_TABLES = %w[ci_builds ci_builds_metadata].freeze
class << self
def enabled?
diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb
new file mode 100644
index 00000000000..583aceba098
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module QueryAnalyzers
+ class PreventSetOperatorMismatch < Base
+ SetOperatorStarError = Class.new(QueryAnalyzerError)
+
+ DETECT_REGEX = /.*SELECT.+(UNION|EXCEPT|INTERSECT)/i
+
+ class << self
+ def enabled?
+ ::Feature::FlipperFeature.table_exists? &&
+ Feature.enabled?(:query_analyzer_gitlab_schema_metrics, type: :ops)
+ end
+
+ def analyze(parsed)
+ return unless requires_detection?(parsed.sql)
+
+ # Only handle SELECT queries.
+ parsed.pg.tree.stmts.each do |stmt|
+ select_stmt = next_select_stmt(stmt)
+ next unless select_stmt
+
+ types = SelectStmt.new(select_stmt).types
+
+ raise SetOperatorStarError if types.any?(Type::INVALID)
+ end
+ end
+
+ private
+
+ def next_select_stmt(node)
+ return unless node.stmt.respond_to?(:select_stmt)
+
+ node.stmt.select_stmt
+ end
+
+ # This not entirely correct and will run true on `SELECT union_station, ...`
+ def requires_detection?(sql)
+ sql.match DETECT_REGEX
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/columns.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/columns.rb
new file mode 100644
index 00000000000..87120b8ffce
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/columns.rb
@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module QueryAnalyzers
+ class PreventSetOperatorMismatch
+ # Columns refer to table columns produced by queries and parts of queries.
+ # If we have `SELECT namespaces.id` then `id` is a column. But, we can also have
+ # `WHERE namespaces.id > 10` and `id` is also a column.
+ #
+ # In static analysis of a SQL query a column source can be ambiguous.
+ # Such as in `SELECT id FROM users, namespaces. In such cases we assume `id` could come from either `users` or
+ # `namespaces`.
+ class Columns
+ class << self
+ # Determine the type of each column in the select statement.
+ # Returns a Set object containing a Types enum.
+ # When an error is found parsing will return immediately.
+ def types(select_stmt)
+ # Forward through any errors when the column refers to a part of the SQL query that is known to include
+ # errors. For example, the column may refer to a column from a CTE that was invalid.
+ return Set.new([Type::INVALID]) if References.errors?(select_stmt.all_references)
+
+ types = Set.new
+
+ # Resolve the type of reference for each target in the select statement.
+ target_list = select_stmt.node.target_list
+ targets = target_list.map(&:res_target)
+ targets.each do |target|
+ target_type = get_target_type(target, select_stmt)
+
+ # A NULL target is of the form:
+ # SELECT NULL::namespaces FROM namespaces
+ types += if Targets.null?(target)
+ # Maintain any errors but otherwise ignore this target.
+ target_type & [Type::INVALID]
+ else
+ target_type
+ end
+ end
+
+ types
+ end
+
+ private
+
+ def get_target_type(target, select_stmt)
+ target_ref_names = Targets.reference_names(target, select_stmt)
+
+ resolved_refs = References.resolved(select_stmt.all_references)
+
+ # Cross reference column references with resolved references.
+ # A resolved reference is part of a SQL query that we were able to analyze already.
+ # A CTE or sub-query would be such a case. The only non-resolvable reference is a table.
+ all_resolved = (target_ref_names - resolved_refs.keys).empty?
+
+ # Is this target `*` such as `SELECT *`.
+ a_star = Targets.a_star?(target)
+
+ if all_resolved
+ # Defer to the reference source types.
+ col_refs = resolved_refs.slice(*target_ref_names)
+ .values
+ .reduce(:union) || Set.new
+
+ if a_star
+ # When * the target forwards through the types of the references.
+ col_refs
+ else
+ # When not * the column is static, but we also forward through any nested errors.
+ (col_refs.to_a & [Type::INVALID]) << Type::STATIC
+ end
+ elsif a_star
+ # This is a * on a table. The * lookup occurs dynamically during query runtime and will
+ # change when the table schema changes.
+ [Type::DYNAMIC]
+ else
+ # This references a column on a table or intermediate result set such as:
+ # SELECT namespaces.id FROM namespaces
+ #
+ # or:
+ # WITH some_cte AS ( ... ) SELECT some_cte.id FROM some_cte
+ [Type::STATIC]
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/common_table_expressions.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/common_table_expressions.rb
new file mode 100644
index 00000000000..0ab58ff7c6f
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/common_table_expressions.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# The CTE in a SELECT can reference CTEs defined by the current scope, but also CTEs defined by earlier scopes.
+# With the following query as an example:
+#
+# WITH some_cte AS (select 1)
+# SELECT *
+# FROM (SELECT * FROM some_cte) subquery
+#
+# The CTE some_cte is visible from within the subquery scope.
+module Gitlab
+ module Database
+ module QueryAnalyzers
+ class PreventSetOperatorMismatch
+ class CommonTableExpressions
+ class << self
+ # Convert CTEs available within this SELECT statement into a set of References.
+ #
+ # @param [PgQuery::Node] node The PgQuery SELECT statement node containing the CTEs.
+ # @param [References] cte_refs Inherited CTEs from scopes that wrap this SELECT statement.
+ def references(node, cte_refs)
+ return cte_refs if node&.with_clause.nil?
+
+ refs = cte_refs.dup
+
+ node.with_clause.ctes.each do |cte|
+ cte_name = name(cte)
+ cte_select_stmt = select_stmt(cte)
+
+ # Resolve the CTE type to dynamic/static/error.
+ refs[cte_name] = if node.with_clause.recursive
+ # Recursive CTEs need special handling to avoid infinite loops.
+ recursive_refs(cte_refs, cte_name, cte_select_stmt)
+ else
+ SelectStmt.new(cte_select_stmt, cte_refs).types
+ end
+ end
+
+ refs
+ end
+
+ private
+
+ def name(cte)
+ cte.common_table_expr.ctename
+ end
+
+ def select_stmt(cte)
+ cte.common_table_expr.ctequery.select_stmt
+ end
+
+ # Return whether the recursive CTE is dynamic/static/error.
+ def recursive_refs(cte_refs, cte_name, select_stmt)
+ # Resolve the non-recursive term before the recursive term.
+ larg_select_stmt = SelectStmt.new(select_stmt.larg, cte_refs)
+ larg_type = larg_select_stmt.types
+ new_cte_refs = cte_refs.merge({ cte_name => larg_type })
+
+ # Now we can resolve the recursive side.
+ rarg_type = SelectStmt.new(select_stmt.rarg, new_cte_refs).types
+
+ final_type = larg_type | rarg_type
+ if final_type.count > 1
+ final_type | [Type::INVALID]
+ else
+ final_type
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/froms.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/froms.rb
new file mode 100644
index 00000000000..c205243694a
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/froms.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module QueryAnalyzers
+ class PreventSetOperatorMismatch
+ class Froms
+ class << self
+ # Parse the FROM part of the SELECT. Construct a mapping of FROM names to their PgQuery node. Recurse any
+ # sub-queries and resolve to a Set of dynamic/static/error.
+ #
+ # Whenever a node is aliased, use the alias name as it's reference and ignore it's original name.
+ #
+ # For example, given:
+ #
+ # SELECT id
+ # FROM namespaces ns
+ #
+ # Return a Hash of { 'ns' => NodeObject }
+ #
+ # @param [PgQuery::Node] node The PgQuery SELECT statement node containing the CTEs.
+ # @param [References] cte_refs Inherited CTEs from scopes that wrap this SELECT statement.
+ #
+ # @return [Hash] name of from references mapped to the node that defines their value, or Set if already
+ # resolved.
+ def references(node, cte_refs)
+ refs = {}
+
+ return refs unless node
+
+ node.from_clause.each do |from|
+ range_var = Node.dig(from, :range_var)
+ range_sq = Node.dig(from, :range_subselect)
+
+ if range_var
+ # FROM some_table
+ # FROM some_table some_alias
+ refs.merge!(range_var_reference(range_var, cte_refs))
+ elsif Node.dig(from, :join_expr)
+ # FROM some_table INNER JOIN other_table
+ range_vars = Node.locate_descendants(from, :range_var)
+ range_vars.each do |range_var|
+ refs.merge!(range_var_reference(range_var, cte_refs))
+ end
+ elsif range_sq
+ # FROM (SELECT ...) some_alias
+ select_stmt = Node.dig(range_sq, :subquery, :select_stmt)
+ refs[range_sq.alias.aliasname] = SelectStmt.new(select_stmt, cte_refs).types
+ end
+ end
+
+ refs
+ end
+
+ private
+
+ def range_var_reference(range_var, cte_refs)
+ relname = Node.dig(range_var, :alias, :aliasname) || range_var.relname
+ reference = cte_refs[range_var.relname] || range_var
+
+ { relname => reference }
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/node.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/node.rb
new file mode 100644
index 00000000000..ee41eaa9d3a
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/node.rb
@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module QueryAnalyzers
+ class PreventSetOperatorMismatch
+ # The Node class allows us to traverse PgQuery nodes with tree like semantics.
+ #
+ # This class balances convenience and performance. The PgQuery nodes are Google::Protobuf::MessageExts which
+ # contain a dynamic set of attributes known as fields. Accessing these fields can cause performance problems
+ # due to the large volume of iterable fields.
+ #
+ # When possible use #dig over the *descendant* methods.
+ #
+ # The filter available to each method reduces the traversed attributes. The default filter only traverses nodes
+ # required to parse for set operator mismatches.
+ class Node
+ class << self
+ include Gitlab::Utils::StrongMemoize
+
+ # The default nodes help speed up traversal. Traversal of other nodes can greatly affect performance.
+ DEFAULT_NODES = %i[
+ a_star
+ alias
+ args
+ column_ref
+ fields
+ func_call
+ join_expr
+ larg
+ range_subselect
+ range_var
+ rarg
+ res_target
+ subquery
+ val
+ ].freeze
+ DEFAULT_FIELD_FILTER = ->(field) { field.is_a?(Integer) || DEFAULT_NODES.include?(field) }.freeze
+
+ # Recurse through children.
+ # The block will yield the child node and the name of that node.
+ # Calling without a block will return an Enumerator.
+ def descendants(node, filter: DEFAULT_FIELD_FILTER, &blk)
+ if blk
+ children(node, filter: filter) do |child_node, child_field|
+ yield(child_node, child_field)
+
+ descendants(child_node, filter: filter, &blk)
+ end
+ nil
+ else
+ enum_for(:descendants, node, filter: filter, &blk)
+ end
+ end
+
+ # Return the first node that matches the field.
+ def locate_descendant(node, field, filter: DEFAULT_FIELD_FILTER)
+ descendants(node, filter: filter).find { |_, child_field| child_field == field }&.first
+ end
+
+ # Return all nodes that match the field.
+ def locate_descendants(node, field, filter: DEFAULT_FIELD_FILTER)
+ descendants(node, filter: filter).select { |_, child_field| child_field == field }.map(&:first)
+ end
+
+ # Like Hash#dig, traverse attributes in sequential order and return the final value.
+ # Return nil if any of the fields are not available.
+ def dig(node, *attrs)
+ obj = node
+ attrs.each do |attr|
+ if obj.respond_to?(attr)
+ obj = obj.public_send(attr) # rubocop:disable GitlabSecurity/PublicSend
+ else
+ obj = nil
+ break
+ end
+ end
+ obj
+ end
+
+ private
+
+ # Interface with a PgQuery result as though it was a tree node.
+ # All elements in a PgQuery result are ancestors of Google::Protobuf::AbstractMessage
+ #
+ # Based off PgQuery's treewalker https://github.com/pganalyze/pg_query/blob/main/lib/pg_query/treewalker.rb
+ def children(node, filter: DEFAULT_FIELD_FILTER, &_blk)
+ attributes = case node
+ when Google::Protobuf::MessageExts
+ descriptor_fields(node.class.descriptor)
+ when Google::Protobuf::RepeatedField
+ node.count.times.to_a
+ end
+
+ attributes.select(&filter).each do |attr|
+ attr_key = attr.is_a?(Symbol) ? attr.to_s : attr
+ child = node[attr_key]
+ next if child.nil?
+
+ yield(child, attr)
+ end
+ end
+
+ def descriptor_fields(descriptor)
+ strong_memoize_with(:descriptor_fields, descriptor) do
+ keys = []
+ descriptor.each do |field|
+ keys << field.name.to_sym
+ end
+ keys
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/references.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/references.rb
new file mode 100644
index 00000000000..ba6e9752905
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/references.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+# References form the base data structure of the PreventSetOperatorMismatch query analyzer.
+#
+# A reference refers to a table, CTE, or other named entity in a SQL query. References are a set of mappings between the
+# name of the reference and the PgQuery node that represents that reference in the parsed tree.
+#
+# Given the SQL:
+#
+# WITH some_cte AS (SELECT 1)
+# SELECT *
+# FROM some_cte, users, namespace ns
+#
+# The reference names would be `some_cte`, `users`, `ns`. The reference values are the nodes in the parse tree that
+# represent that reference:
+# - some_cte: the common table expression node
+# - users: nil, being a table
+# - ns: nil, being a table, but importantly we use the alias name
+#
+# A reference can be "resolved". A resolved reference value is a Set of Types. The reference value was a select
+# statement that has since been parsed.
+module Gitlab
+ module Database
+ module QueryAnalyzers
+ class PreventSetOperatorMismatch
+ class References
+ class << self
+ # All references that have already been parsed to determine static/dynamic/error state.
+ # @param [Hash] refs A Hash of reference names mapped to the parse tree node or resolved Set of Types.
+ def resolved(refs)
+ refs.select { |_name, ref| ref.is_a?(Set) }
+ end
+
+ # All references that have not been parsed to determine static/dynamic/error state.
+ # @param [Hash] refs A Hash of reference names mapped to the parse tree node or resolved Set of Types.
+ def unresolved(refs)
+ refs.select { |_name, ref| unresolved?(ref) }
+ end
+
+ # Whether any currently resolved references have resulted in an error state.
+ # @param [Hash] refs A Hash of reference names mapped to the parse tree node or resolved Set of Types.
+ def errors?(refs)
+ resolved(refs).any? { |_, values| values.include?(Type::INVALID) }
+ end
+
+ private
+
+ def resolved?(ref)
+ ref.is_a?(Set)
+ end
+
+ def unresolved?(ref)
+ !resolved?(ref) && table?(ref)
+ end
+
+ def table?(ref)
+ !ref.is_a?(PgQuery::RangeVar)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/select_stmt.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/select_stmt.rb
new file mode 100644
index 00000000000..bdbcc49f63f
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/select_stmt.rb
@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module QueryAnalyzers
+ class PreventSetOperatorMismatch
+ class SelectStmt
+ include Gitlab::Utils::StrongMemoize
+
+ attr_reader :node, :cte_references, :all_references
+
+ # @param [PgQuery::SelectStmt] node The PgQuery node of the select statement.
+ # @param [Hash] inherited_cte_references CTE References available to the select statement.
+ def initialize(node, inherited_cte_references = {})
+ @node = node
+ @cte_references = CommonTableExpressions.references(node, inherited_cte_references)
+ from_references = Froms.references(node, cte_references)
+ @all_references = from_references.merge(cte_references)
+ end
+
+ # returns Set of Types.
+ #
+ # STATIC - queries that don't require a database schema lookup. E.g. `SELECT users.id FROM users`
+ # DYNAMIC - queries that require a database schema lookup. E.g. `SELECT users.* FROM users`
+ # INVALID - set operator queries that mix static and dynamic queries.
+ def types
+ if set_operator?
+ resolve_set_operator_select_types
+ else
+ resolve_normal_select_types
+ end
+ end
+
+ private
+
+ # Standard SELECT, not a set operator (UNION/INTERSECT/EXCEPT)
+ def resolve_normal_select_types
+ # Cross reference resolved sources with what is requested by the SELECT.
+ types = Columns.types(self)
+
+ # Mixed dynamic and static queries can be normalized to simply dynamic queries for the purposes of
+ # detecting mismatched set operator parts.
+ types.delete(Type::STATIC) if types.include?(Type::DYNAMIC)
+
+ types
+ end
+
+ # Set operator (UNION/INTERSECT/EXCEPT)
+ def resolve_set_operator_select_types
+ types = Set.new
+
+ # Recurse each set operator part as a SELECT statement.
+ # select statement part => type
+ set_operator_parts do |part|
+ types += SelectStmt.new(part, cte_references).types
+ end
+
+ types << Type::INVALID if types.count > 1
+
+ types
+ end
+
+ def set_operator?
+ !(node.respond_to?(:op) && node.op == :SETOP_NONE)
+ end
+
+ SET_OPERATOR_PART_LOCATIONS = %i[larg rarg].freeze
+ private_constant :SET_OPERATOR_PART_LOCATIONS
+
+ def set_operator_parts(&_blk)
+ return unless node
+
+ yield node if node.op == :SETOP_NONE
+ yield node.larg if node.larg && node.larg.op == :SETOP_NONE
+ yield node.rarg if node.rarg && node.rarg.op == :SETOP_NONE
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/targets.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/targets.rb
new file mode 100644
index 00000000000..99db368efcb
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/targets.rb
@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+# Targets refer to SELECT columns but also JOIN fields, etc.
+# A target can have a qualifying reference to some other entity like a table or CTE.
+module Gitlab
+ module Database
+ module QueryAnalyzers
+ class PreventSetOperatorMismatch
+ class Targets
+ class << self
+ # Return the reference names used by the given target.
+ #
+ # For example:
+ # `SELECT users.id` would return ['users']
+ # `SELECT * FROM users, namespaces` would return ['users', 'namespaces']
+ def reference_names(target, select_stmt)
+ # Parse all targets to determine what is referenced.
+ fields = fields(target)
+ case fields.count
+ when 0
+ literal_ref_names(target, select_stmt)
+ when 1
+ unqualified_ref_names(fields, select_stmt)
+ else
+ # The target is qualified such as SELECT reference.id
+ field_ref = fields[fields.count - 2]
+ [field_ref.string.sval]
+ end
+ end
+
+ # True when `SELECT *`
+ def a_star?(target)
+ Node.locate_descendant(target, :a_star)
+ end
+
+ # Null targets are used to produce "polymorphic" query result sets that can be aggregated through a UNION
+ # without having to worry about mismatched columns.
+ #
+ # A null target would be something like:
+ # SELECT NULL::namespaces FROM namespaces
+ def null?(target)
+ target&.val&.type_cast&.arg&.a_const&.isnull
+ end
+
+ private
+
+ def literal_ref_names(target, select_stmt)
+ # The target is unqualified and is not part of a column_ref, such as in `SELECT 1`.
+ # These include targets like literals, functions, and subselects.
+ sub_select_stmt = subselect_select_stmt(target)
+ if sub_select_stmt
+ name = (target.name.presence || "loc_#{target.location}")
+ # The select is anonymous, so we provide a name.
+ k = "#{name}_subselect"
+ # Force parsing of the select.
+ # We don't care about the static/dynamic nature in this case, but we do need to parse for
+ # any nested error states.
+ sub_select = SelectStmt.new(sub_select_stmt, select_stmt.cte_references)
+ select_stmt.all_references[k] = sub_select.types
+ [k]
+ else
+ # TODO we need to parse function references. Assuming no sources for now.
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/428102
+ []
+ end
+ end
+
+ def unqualified_ref_names(fields, select_stmt)
+ # The target is unqualified, but is part of a column_ref.
+ # E.g. `SELECT id FROM namespaces` or `SELECT namespaces FROM namespaces`
+
+ # Otherwise, check all FROM/JOIN/CTE entries.
+ field = fields[0]
+ field_sval = field&.string&.sval
+ if field_sval && select_stmt.all_references.key?(field_sval)
+ # SELECT some_table_name
+ [field.string.sval]
+ else
+ # SELECT *
+ # SELECT some_column
+ select_stmt.all_references.keys
+ end
+ end
+
+ def fields(target)
+ Node.locate_descendants(target, :fields).flatten
+ end
+
+ def subselect_select_stmt(target)
+ Node.dig(target, :val, :sub_link, :subselect, :select_stmt)
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/type.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/type.rb
new file mode 100644
index 00000000000..5988f963827
--- /dev/null
+++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/type.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ module QueryAnalyzers
+ class PreventSetOperatorMismatch
+ # An enumerated set of constants that represent the state of the parse.
+ module Type
+ STATIC = :static
+ DYNAMIC = :dynamic
+ INVALID = :invalid
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/schema_cache_with_renamed_table.rb b/lib/gitlab/database/schema_cache_with_renamed_table.rb
index 6da76803f7c..e110fc44b7b 100644
--- a/lib/gitlab/database/schema_cache_with_renamed_table.rb
+++ b/lib/gitlab/database/schema_cache_with_renamed_table.rb
@@ -11,26 +11,26 @@ module Gitlab
clear_renamed_tables_cache!
end
- def clear_data_source_cache!(name)
- super(name)
+ def clear_data_source_cache!(connection, table_name)
+ super(connection, table_name)
clear_renamed_tables_cache!
end
- def primary_keys(table_name)
- super(underlying_table(table_name))
+ def primary_keys(connection, table_name)
+ super(connection, underlying_table(table_name))
end
- def columns(table_name)
- super(underlying_table(table_name))
+ def columns(connection, table_name)
+ super(connection, underlying_table(table_name))
end
- def columns_hash(table_name)
- super(underlying_table(table_name))
+ def columns_hash(connection, table_name)
+ super(connection, underlying_table(table_name))
end
- def indexes(table_name)
- super(underlying_table(table_name))
+ def indexes(connection, table_name)
+ super(connection, underlying_table(table_name))
end
private
@@ -40,7 +40,7 @@ module Gitlab
end
def renamed_tables_cache
- @renamed_tables ||= Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name|
+ @renamed_tables ||= Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, _new_name|
connection.view_exists?(old_name)
end
end
diff --git a/lib/gitlab/database/schema_cache_with_renamed_table_legacy.rb b/lib/gitlab/database/schema_cache_with_renamed_table_legacy.rb
new file mode 100644
index 00000000000..acc9bbd0aff
--- /dev/null
+++ b/lib/gitlab/database/schema_cache_with_renamed_table_legacy.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Database
+ # This is a legacy extension targeted at Rails versions prior to 7.1
+ # In Rails 7.1, the method parameters have been changed to (connection, table_name)
+ module SchemaCacheWithRenamedTableLegacy
+ # Override methods in ActiveRecord::ConnectionAdapters::SchemaCache
+
+ def clear!
+ super
+
+ clear_renamed_tables_cache!
+ end
+
+ def clear_data_source_cache!(name)
+ super(name)
+
+ clear_renamed_tables_cache!
+ end
+
+ def primary_keys(table_name)
+ super(underlying_table(table_name))
+ end
+
+ def columns(table_name)
+ super(underlying_table(table_name))
+ end
+
+ def columns_hash(table_name)
+ super(underlying_table(table_name))
+ end
+
+ def indexes(table_name)
+ super(underlying_table(table_name))
+ end
+
+ private
+
+ def underlying_table(table_name)
+ renamed_tables_cache.fetch(table_name, table_name)
+ end
+
+ def renamed_tables_cache
+ @renamed_tables ||= Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, _new_name|
+ connection.view_exists?(old_name)
+ end
+ end
+
+ def clear_renamed_tables_cache!
+ @renamed_tables = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb
index aa880b709fe..608dea9e3c5 100644
--- a/lib/gitlab/database/tables_locker.rb
+++ b/lib/gitlab/database/tables_locker.rb
@@ -3,7 +3,7 @@
module Gitlab
module Database
class TablesLocker
- GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo].freeze
+ GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo gitlab_jh].freeze
def initialize(logger: nil, dry_run: false, include_partitions: true)
@logger = logger
diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb
index f91146fff3d..5394dee6fec 100644
--- a/lib/gitlab/database/tables_truncate.rb
+++ b/lib/gitlab/database/tables_truncate.rb
@@ -3,7 +3,7 @@
module Gitlab
module Database
class TablesTruncate
- GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo gitlab_embedding].freeze
+ GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo gitlab_embedding gitlab_jh].freeze
def initialize(database_name:, min_batch_size: 5, logger: nil, until_table: nil, dry_run: false)
@database_name = database_name
diff --git a/lib/gitlab/discussions_diff/file_collection.rb b/lib/gitlab/discussions_diff/file_collection.rb
index 60b3a1738f1..3d1f7ab86b3 100644
--- a/lib/gitlab/discussions_diff/file_collection.rb
+++ b/lib/gitlab/discussions_diff/file_collection.rb
@@ -25,8 +25,9 @@ module Gitlab
#
# - Highlight cache is written just for uncached diff files
# - The cache content is not updated (there's no need to do so)
- def load_highlight
- ids = highlightable_collection_ids
+ # - Load only the related diff note ids
+ def load_highlight(diff_note_ids: nil)
+ ids = highlightable_collection_ids(diff_note_ids)
return if ids.empty?
cached_content = read_cache(ids)
@@ -47,8 +48,13 @@ module Gitlab
private
- def highlightable_collection_ids
- each.with_object([]) { |file, memo| memo << file.id unless file.resolved_at }
+ def highlightable_collection_ids(diff_note_ids)
+ each.with_object([]) do |file, memo|
+ # We ignore if file is resolved, or not part of the highlight requested notes
+ next if file.resolved_at || (diff_note_ids.present? && diff_note_ids.exclude?(file.diff_note_id))
+
+ memo << file.id
+ end
end
def read_cache(ids)
diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb
index ebc4e9c2c8c..e3249b143c8 100644
--- a/lib/gitlab/email/handler/service_desk_handler.rb
+++ b/lib/gitlab/email/handler/service_desk_handler.rb
@@ -38,7 +38,7 @@ module Gitlab
create_issue_or_note
if from_address
- add_email_participant
+ add_email_participants
send_thank_you_email unless reply_email?
end
end
@@ -215,6 +215,10 @@ module Gitlab
end
strong_memoize_attr :to_address
+ def cc_addresses
+ mail.cc || []
+ end
+
def can_handle_legacy_format?
project_path && project_path.include?('/') && !mail_key.include?('+')
end
@@ -223,11 +227,33 @@ module Gitlab
Users::Internal.support_bot
end
- def add_email_participant
+ def add_email_participants
return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project)
@issue.issue_email_participants.create(email: from_address)
+
+ add_external_participants_from_cc
+ end
+
+ def add_external_participants_from_cc
+ return if project.service_desk_setting.nil?
+ return unless project.service_desk_setting.add_external_participants_from_cc?
+
+ cc_addresses.each do |email|
+ next if service_desk_addresses.include?(email)
+
+ @issue.issue_email_participants.create!(email: email)
+ end
+ end
+
+ def service_desk_addresses
+ [
+ project.service_desk_incoming_address,
+ project.service_desk_alias_address,
+ project.service_desk_custom_address
+ ].compact
end
+ strong_memoize_attr :service_desk_addresses
end
end
end
diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb
index 7d47bfe88fe..1a7a2fba2f3 100644
--- a/lib/gitlab/emoji.rb
+++ b/lib/gitlab/emoji.rb
@@ -6,7 +6,7 @@ module Gitlab
# When updating emoji assets increase the version below
# and update the version number in `app/assets/javascripts/emoji/index.js`
- EMOJI_VERSION = 2
+ EMOJI_VERSION = 3
# Return a Pathname to emoji's current versioned folder
#
diff --git a/lib/gitlab/encrypted_command_base.rb b/lib/gitlab/encrypted_command_base.rb
index b35c28b85cd..679d9d8e31a 100644
--- a/lib/gitlab/encrypted_command_base.rb
+++ b/lib/gitlab/encrypted_command_base.rb
@@ -7,12 +7,12 @@ module Gitlab
EDIT_COMMAND_NAME = "base"
class << self
- def encrypted_secrets
+ def encrypted_secrets(**args)
raise NotImplementedError
end
- def write(contents)
- encrypted = encrypted_secrets
+ def write(contents, args: {})
+ encrypted = encrypted_secrets(**args)
return unless validate_config(encrypted)
validate_contents(contents)
@@ -25,8 +25,8 @@ module Gitlab
warn "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?"
end
- def edit
- encrypted = encrypted_secrets
+ def edit(args: {})
+ encrypted = encrypted_secrets(**args)
return unless validate_config(encrypted)
if ENV["EDITOR"].blank?
@@ -58,8 +58,8 @@ module Gitlab
temp_file&.unlink
end
- def show
- encrypted = encrypted_secrets
+ def show(args: {})
+ encrypted = encrypted_secrets(**args)
return unless validate_config(encrypted)
puts encrypted.read.presence || "File '#{encrypted.content_path}' does not exist. Use `gitlab-rake #{self::EDIT_COMMAND_NAME}` to change that."
diff --git a/lib/gitlab/encrypted_configuration.rb b/lib/gitlab/encrypted_configuration.rb
index 6b64281e631..5ead57e17fd 100644
--- a/lib/gitlab/encrypted_configuration.rb
+++ b/lib/gitlab/encrypted_configuration.rb
@@ -30,7 +30,7 @@ module Gitlab
end
def initialize(content_path: nil, base_key: nil, previous_keys: [])
- @content_path = Pathname.new(content_path).yield_self { |path| path.symlink? ? path.realpath : path } if content_path
+ @content_path = Pathname.new(content_path).then { |path| path.symlink? ? path.realpath : path } if content_path
@key = self.class.generate_key(base_key) if base_key
@previous_keys = previous_keys
end
diff --git a/lib/gitlab/encrypted_ldap_command.rb b/lib/gitlab/encrypted_ldap_command.rb
index 5e1eabe7ec6..442c675f19e 100644
--- a/lib/gitlab/encrypted_ldap_command.rb
+++ b/lib/gitlab/encrypted_ldap_command.rb
@@ -1,6 +1,5 @@
# frozen_string_literal: true
-# rubocop:disable Rails/Output
module Gitlab
class EncryptedLdapCommand < EncryptedCommandBase
DISPLAY_NAME = "LDAP"
@@ -21,4 +20,3 @@ module Gitlab
end
end
end
-# rubocop:enable Rails/Output
diff --git a/lib/gitlab/encrypted_redis_command.rb b/lib/gitlab/encrypted_redis_command.rb
new file mode 100644
index 00000000000..608edcdb950
--- /dev/null
+++ b/lib/gitlab/encrypted_redis_command.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# rubocop:disable Rails/Output
+module Gitlab
+ class EncryptedRedisCommand < EncryptedCommandBase
+ DISPLAY_NAME = "Redis"
+ EDIT_COMMAND_NAME = "gitlab:redis:secret:edit"
+
+ class << self
+ def all_redis_instance_class_names
+ Gitlab::Redis::ALL_CLASSES.map do |c|
+ normalized_instance_name(c)
+ end
+ end
+
+ def normalized_instance_name(instance)
+ if instance.is_a?(Class)
+ # Gitlab::Redis::SharedState => sharedstate
+ instance.name.demodulize.to_s.downcase
+ else
+ # Drop all hyphens, underscores, and spaces from the name
+ # eg.: shared_state => sharedstate
+ instance.gsub(/[-_ ]/, '').downcase
+ end
+ end
+
+ def encrypted_secrets(**args)
+ if args[:instance_name]
+ instance_class = Gitlab::Redis::ALL_CLASSES.find do |instance|
+ normalized_instance_name(instance) == normalized_instance_name(args[:instance_name])
+ end
+
+ unless instance_class
+ error_message = <<~MSG
+ Specified instance name #{args[:instance_name]} does not exist.
+ The available instances are #{all_redis_instance_class_names.join(', ')}."
+ MSG
+
+ raise error_message
+ end
+ else
+ instance_class = Gitlab::Redis::Cache
+ end
+
+ instance_class.encrypted_secrets
+ end
+
+ def encrypted_file_template
+ <<~YAML
+ # password: '123'
+ YAML
+ end
+ end
+ end
+end
+# rubocop:enable Rails/Output
diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb
index 13959f6aa68..ef8f2d4d61b 100644
--- a/lib/gitlab/file_detector.rb
+++ b/lib/gitlab/file_detector.rb
@@ -21,7 +21,7 @@ module Gitlab
# Configuration files
gitignore: '.gitignore',
- gitlab_ci: '.gitlab-ci.yml',
+ gitlab_ci: ::Ci::Pipeline::DEFAULT_CONFIG_PATH,
route_map: '.gitlab/route-map.yml',
# Dependency files
diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb
index 3d2bde6f0a7..e134fb31879 100644
--- a/lib/gitlab/git/blame.rb
+++ b/lib/gitlab/git/blame.rb
@@ -4,7 +4,6 @@ module Gitlab
module Git
class Blame
include Gitlab::EncodingHelper
- include Gitlab::Git::WrapsGitalyErrors
attr_reader :lines, :blames, :range
@@ -35,11 +34,9 @@ module Gitlab
end
def fetch_raw_blame
- wrapped_gitaly_errors do
- @repo.gitaly_commit_client.raw_blame(@sha, @path, range: range_spec)
- end
- # Return empty result when blame range is out-of-range
+ @repo.gitaly_commit_client.raw_blame(@sha, @path, range: range_spec)
rescue ArgumentError
+ # Return an empty result when the blame range is out-of-range or path is not found
""
end
diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb
index ae90291c0a3..3744c81f51d 100644
--- a/lib/gitlab/git/blob.rb
+++ b/lib/gitlab/git/blob.rb
@@ -230,5 +230,3 @@ module Gitlab
end
end
end
-
-Gitlab::Git::Blob.singleton_class.prepend Gitlab::Git::RuggedImpl::Blob::ClassMethods
diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb
index 571dde6fcfc..1086ea45a7a 100644
--- a/lib/gitlab/git/commit.rb
+++ b/lib/gitlab/git/commit.rb
@@ -5,7 +5,6 @@ module Gitlab
module Git
class Commit
include Gitlab::EncodingHelper
- prepend Gitlab::Git::RuggedImpl::Commit
extend Gitlab::Git::WrapsGitalyErrors
include Gitlab::Utils::StrongMemoize
@@ -502,5 +501,3 @@ module Gitlab
end
end
end
-
-Gitlab::Git::Commit.singleton_class.prepend Gitlab::Git::RuggedImpl::Commit::ClassMethods
diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb
index 4a09f866db4..205dd5be35a 100644
--- a/lib/gitlab/git/ref.rb
+++ b/lib/gitlab/git/ref.rb
@@ -4,7 +4,6 @@ module Gitlab
module Git
class Ref
include Gitlab::EncodingHelper
- include Gitlab::Git::RuggedImpl::Ref
# Branch or tag name
# without "refs/tags|heads" prefix
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index a98cf95edf4..db6e6b4d00b 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -11,7 +11,6 @@ module Gitlab
include Gitlab::Git::WrapsGitalyErrors
include Gitlab::EncodingHelper
include Gitlab::Utils::StrongMemoize
- prepend Gitlab::Git::RuggedImpl::Repository
SEARCH_CONTEXT_LINES = 3
REV_LIST_COMMIT_LIMIT = 2_000
diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb
deleted file mode 100644
index dc869ff5279..00000000000
--- a/lib/gitlab/git/rugged_impl/blob.rb
+++ /dev/null
@@ -1,107 +0,0 @@
-# frozen_string_literal: true
-
-# NOTE: This code is legacy. Do not add/modify code here unless you have
-# discussed with the Gitaly team. See
-# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code
-# for more details.
-
-module Gitlab
- module Git
- module RuggedImpl
- module Blob
- module ClassMethods
- extend ::Gitlab::Utils::Override
- include Gitlab::Git::RuggedImpl::UseRugged
-
- override :tree_entry
- def tree_entry(repository, sha, path, limit)
- if use_rugged?(repository, :rugged_tree_entry)
- execute_rugged_call(:rugged_tree_entry, repository, sha, path, limit)
- else
- super
- end
- end
-
- private
-
- def rugged_tree_entry(repository, sha, path, limit)
- return unless path
-
- # Strip any leading / characters from the path
- path = path.sub(%r{\A/*}, '')
-
- rugged_commit = repository.lookup(sha)
- root_tree = rugged_commit.tree
-
- blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/'))
-
- return unless blob_entry
-
- if blob_entry[:type] == :commit
- submodule_blob(blob_entry, path, sha)
- else
- blob = repository.lookup(blob_entry[:oid])
-
- if blob
- new(
- id: blob.oid,
- name: blob_entry[:name],
- size: blob.size,
- # Rugged::Blob#content is expensive; don't call it if we don't have to.
- data: limit == 0 ? '' : blob.content(limit),
- mode: blob_entry[:filemode].to_s(8),
- path: path,
- commit_id: sha,
- binary: blob.binary?
- )
- end
- end
- rescue Rugged::ReferenceError
- nil
- end
-
- # Recursive search of blob id by path
- #
- # Ex.
- # blog/ # oid: 1a
- # app/ # oid: 2a
- # models/ # oid: 3a
- # file.rb # oid: 4a
- #
- #
- # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a'
- #
- def find_entry_by_path(repository, root_id, *path_parts)
- root_tree = repository.lookup(root_id)
-
- entry = root_tree.find do |entry|
- entry[:name] == path_parts[0]
- end
-
- return unless entry
-
- if path_parts.size > 1
- return unless entry[:type] == :tree
-
- path_parts.shift
- find_entry_by_path(repository, entry[:oid], *path_parts)
- else
- [:blob, :commit].include?(entry[:type]) ? entry : nil
- end
- end
-
- def submodule_blob(blob_entry, path, sha)
- new(
- id: blob_entry[:oid],
- name: blob_entry[:name],
- size: 0,
- data: '',
- path: path,
- commit_id: sha
- )
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb
deleted file mode 100644
index cf547414b0d..00000000000
--- a/lib/gitlab/git/rugged_impl/commit.rb
+++ /dev/null
@@ -1,115 +0,0 @@
-# frozen_string_literal: true
-
-# NOTE: This code is legacy. Do not add/modify code here unless you have
-# discussed with the Gitaly team. See
-# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code
-# for more details.
-
-# rubocop:disable Gitlab/ModuleWithInstanceVariables
-module Gitlab
- module Git
- module RuggedImpl
- module Commit
- module ClassMethods
- extend ::Gitlab::Utils::Override
- include Gitlab::Git::RuggedImpl::UseRugged
-
- def rugged_find(repo, commit_id)
- obj = repo.rev_parse_target(commit_id)
-
- obj.is_a?(::Rugged::Commit) ? obj : nil
- rescue ::Rugged::Error
- nil
- end
-
- # This needs to return an array of Gitlab::Git:Commit objects
- # instead of Rugged::Commit objects to ensure upstream models
- # operate on a consistent interface. Unlike
- # Gitlab::Git::Commit.find, Gitlab::Git::Commit.batch_by_oid
- # doesn't attempt to decorate the result.
- def rugged_batch_by_oid(repo, oids)
- oids.map { |oid| rugged_find(repo, oid) }
- .compact
- .map { |commit| decorate(repo, commit) }
- # Match Gitaly's list_commits_by_oid behavior
- rescue ::Gitlab::Git::Repository::NoRepository
- []
- end
-
- override :find_commit
- def find_commit(repo, commit_id)
- if use_rugged?(repo, :rugged_find_commit)
- execute_rugged_call(:rugged_find, repo, commit_id)
- else
- super
- end
- end
-
- override :batch_by_oid
- def batch_by_oid(repo, oids)
- if use_rugged?(repo, :rugged_list_commits_by_oid)
- execute_rugged_call(:rugged_batch_by_oid, repo, oids)
- else
- super
- end
- end
- end
-
- extend ::Gitlab::Utils::Override
- include Gitlab::Git::RuggedImpl::UseRugged
-
- override :init_commit
- def init_commit(raw_commit)
- case raw_commit
- when ::Rugged::Commit
- init_from_rugged(raw_commit)
- else
- super
- end
- end
-
- override :commit_tree_entry
- def commit_tree_entry(path)
- if use_rugged?(@repository, :rugged_commit_tree_entry)
- execute_rugged_call(:rugged_tree_entry, path)
- else
- super
- end
- end
-
- # Is this the same as Blob.find_entry_by_path ?
- def rugged_tree_entry(path)
- rugged_commit.tree.path(path)
- rescue Rugged::TreeError
- nil
- end
-
- def rugged_commit
- @rugged_commit ||= if raw_commit.is_a?(Rugged::Commit)
- raw_commit
- else
- @repository.rev_parse_target(id)
- end
- end
-
- def init_from_rugged(commit)
- author = commit.author
- committer = commit.committer
-
- @raw_commit = commit
- @id = commit.oid
- @message = commit.message
- @authored_date = author[:time]
- @committed_date = committer[:time]
- @author_name = author[:name]
- @author_email = author[:email]
- @committer_name = committer[:name]
- @committer_email = committer[:email]
- @parent_ids = commit.parents.map(&:oid)
- @trailers = Hash[commit.trailers]
- end
- end
- end
- end
-end
-# rubocop:enable Gitlab/ModuleWithInstanceVariables
diff --git a/lib/gitlab/git/rugged_impl/ref.rb b/lib/gitlab/git/rugged_impl/ref.rb
deleted file mode 100644
index b553e82dc47..00000000000
--- a/lib/gitlab/git/rugged_impl/ref.rb
+++ /dev/null
@@ -1,20 +0,0 @@
-# frozen_string_literal: true
-
-# NOTE: This code is legacy. Do not add/modify code here unless you have
-# discussed with the Gitaly team. See
-# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code
-# for more details.
-
-module Gitlab
- module Git
- module RuggedImpl
- module Ref
- def self.dereference_object(object)
- object = object.target while object.is_a?(::Rugged::Tag::Annotation)
-
- object
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb
deleted file mode 100644
index cd4eefa158e..00000000000
--- a/lib/gitlab/git/rugged_impl/repository.rb
+++ /dev/null
@@ -1,79 +0,0 @@
-# frozen_string_literal: true
-
-# NOTE: This code is legacy. Do not add/modify code here unless you have
-# discussed with the Gitaly team. See
-# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code
-# for more details.
-
-# rubocop:disable Gitlab/ModuleWithInstanceVariables
-module Gitlab
- module Git
- module RuggedImpl
- module Repository
- extend ::Gitlab::Utils::Override
- include Gitlab::Git::RuggedImpl::UseRugged
-
- FEATURE_FLAGS = %i[rugged_find_commit rugged_tree_entries rugged_tree_entry rugged_commit_is_ancestor rugged_commit_tree_entry rugged_list_commits_by_oid].freeze
-
- def alternate_object_directories
- relative_object_directories.map { |d| File.join(path, d) }
- end
-
- ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[
- GIT_OBJECT_DIRECTORY_RELATIVE
- GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE
- ].freeze
-
- def relative_object_directories
- Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact
- end
-
- def rugged
- @rugged ||= ::Rugged::Repository.new(path, alternates: alternate_object_directories)
- rescue ::Rugged::RepositoryError, ::Rugged::OSError
- raise ::Gitlab::Git::Repository::NoRepository, 'no repository for such path'
- end
-
- def cleanup
- @rugged&.close
- end
-
- # Return the object that +revspec+ points to. If +revspec+ is an
- # annotated tag, then return the tag's target instead.
- def rev_parse_target(revspec)
- obj = rugged.rev_parse(revspec)
- Ref.dereference_object(obj)
- end
-
- override :ancestor?
- def ancestor?(from, to)
- if use_rugged?(self, :rugged_commit_is_ancestor)
- execute_rugged_call(:rugged_is_ancestor?, from, to)
- else
- super
- end
- end
-
- def rugged_is_ancestor?(ancestor_id, descendant_id)
- return false if ancestor_id.nil? || descendant_id.nil?
-
- rugged_merge_base(ancestor_id, descendant_id) == ancestor_id
- rescue Rugged::OdbError
- false
- end
-
- def rugged_merge_base(from, to)
- rugged.merge_base(from, to)
- rescue Rugged::ReferenceError
- nil
- end
-
- # Lookup for rugged object by oid or ref name
- def lookup(oid_or_ref_name)
- rev_parse_target(oid_or_ref_name)
- end
- end
- end
- end
-end
-# rubocop:enable Gitlab/ModuleWithInstanceVariables
diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb
deleted file mode 100644
index bc3ff01e1e2..00000000000
--- a/lib/gitlab/git/rugged_impl/tree.rb
+++ /dev/null
@@ -1,147 +0,0 @@
-# frozen_string_literal: true
-
-# NOTE: This code is legacy. Do not add/modify code here unless you have
-# discussed with the Gitaly team. See
-# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code
-# for more details.
-
-module Gitlab
- module Git
- module RuggedImpl
- module Tree
- module ClassMethods
- extend ::Gitlab::Utils::Override
- include Gitlab::Git::RuggedImpl::UseRugged
-
- TREE_SORT_ORDER = { tree: 0, blob: 1, commit: 2 }.freeze
-
- override :tree_entries
- def tree_entries(repository, sha, path, recursive, skip_flat_paths, rescue_not_found, pagination_params = nil)
- if use_rugged?(repository, :rugged_tree_entries)
- entries = execute_rugged_call(
- :tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive, skip_flat_paths)
-
- if pagination_params
- paginated_response(entries, pagination_params[:limit], pagination_params[:page_token].to_s)
- else
- [entries, nil]
- end
- else
- super
- end
- end
-
- # Rugged version of TreePagination in Go: https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3611
- def paginated_response(entries, limit, token)
- total_entries = entries.count
-
- return [[], nil] if limit == 0 || limit.blank?
-
- entries = Gitlab::Utils.stable_sort_by(entries) { |x| TREE_SORT_ORDER[x.type] }
-
- if token.blank?
- index = 0
- else
- index = entries.index { |entry| entry.id == token }
-
- raise Gitlab::Git::CommandError, "could not find starting OID: #{token}" if index.nil?
-
- index += 1
- end
-
- return [entries[index..], nil] if limit < 0
-
- last_index = index + limit
- result = entries[index...last_index]
-
- if last_index < total_entries
- cursor = Gitaly::PaginationCursor.new(next_cursor: result.last.id)
- end
-
- [result, cursor]
- end
-
- def tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive, skip_flat_paths)
- tree_entries_from_rugged(repository, sha, path, recursive).tap do |entries|
- # This was an optimization to reduce N+1 queries for Gitaly
- # (https://gitlab.com/gitlab-org/gitaly/issues/530).
- rugged_populate_flat_path(repository, sha, path, entries) unless skip_flat_paths
- end
- end
-
- def tree_entries_from_rugged(repository, sha, path, recursive)
- current_path_entries = get_tree_entries_from_rugged(repository, sha, path)
- ordered_entries = []
-
- current_path_entries.each do |entry|
- ordered_entries << entry
-
- if recursive && entry.dir?
- ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true))
- end
- end
-
- ordered_entries
- end
-
- def rugged_populate_flat_path(repository, sha, path, entries)
- entries.each do |entry|
- entry.flat_path = entry.path
-
- next unless entry.dir?
-
- entry.flat_path =
- if path
- File.join(path, rugged_flatten_tree(repository, sha, entry, path))
- else
- rugged_flatten_tree(repository, sha, entry, path)
- end
- end
- end
-
- # Returns the relative path of the first subdir that doesn't have only one directory descendant
- def rugged_flatten_tree(repository, sha, tree, root_path)
- subtree = tree_entries_from_rugged(repository, sha, tree.path, false)
-
- if subtree.count == 1 && subtree.first.dir?
- File.join(tree.name, rugged_flatten_tree(repository, sha, subtree.first, root_path))
- else
- tree.name
- end
- end
-
- def get_tree_entries_from_rugged(repository, sha, path)
- commit = repository.lookup(sha)
- root_tree = commit.tree
-
- tree = if path
- id = find_id_by_path(repository, root_tree.oid, path)
- if id
- repository.lookup(id)
- else
- []
- end
- else
- root_tree
- end
-
- tree.map do |entry|
- current_path = path ? File.join(path, entry[:name]) : entry[:name]
-
- new(
- id: entry[:oid],
- name: entry[:name],
- type: entry[:type],
- mode: entry[:filemode].to_s(8),
- path: current_path,
- commit_id: sha
- )
- end
- rescue Rugged::ReferenceError
- []
- end
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb
deleted file mode 100644
index 57cced97d02..00000000000
--- a/lib/gitlab/git/rugged_impl/use_rugged.rb
+++ /dev/null
@@ -1,50 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Git
- module RuggedImpl
- module UseRugged
- def use_rugged?(_, _)
- false
- end
-
- def execute_rugged_call(method_name, *args)
- Gitlab::GitalyClient::StorageSettings.allow_disk_access do
- start = Gitlab::Metrics::System.monotonic_time
-
- result = send(method_name, *args) # rubocop:disable GitlabSecurity/PublicSend
-
- duration = Gitlab::Metrics::System.monotonic_time - start
-
- if Gitlab::RuggedInstrumentation.active?
- Gitlab::RuggedInstrumentation.increment_query_count
- Gitlab::RuggedInstrumentation.add_query_time(duration)
-
- Gitlab::RuggedInstrumentation.add_call_details(
- feature: method_name,
- args: args,
- duration: duration,
- backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller))
- end
-
- result
- end
- end
-
- def running_puma_with_multiple_threads?
- return false unless Gitlab::Runtime.puma?
-
- ::Puma.respond_to?(:cli_config) && ::Puma.cli_config.options[:max_threads] > 1
- end
-
- def rugged_feature_keys
- Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS
- end
-
- def rugged_enabled_through_feature_flag?
- false
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb
index 6e97e412b91..4747ab55c63 100644
--- a/lib/gitlab/git/tree.rb
+++ b/lib/gitlab/git/tree.rb
@@ -12,9 +12,6 @@ module Gitlab
class << self
# Get list of tree objects
# for repository based on commit sha and path
- # Uses rugged for raw objects
- #
- # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320
def where(
repository, sha, path = nil, recursive = false, skip_flat_paths = true, rescue_not_found = true,
pagination_params = nil)
@@ -110,5 +107,3 @@ module Gitlab
end
end
end
-
-Gitlab::Git::Tree.singleton_class.prepend Gitlab::Git::RuggedImpl::Tree::ClassMethods
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 45283d51b1b..72016aa1183 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -101,7 +101,7 @@ module Gitlab
end
def guest_can_download?
- Guest.can?(download_ability, container)
+ ::Users::Anonymous.can?(download_ability, container)
end
def deploy_key_can_download_code?
@@ -395,7 +395,7 @@ module Gitlab
user.can?(:read_project, project)
elsif ci?
false
- end || Guest.can?(:read_project, project)
+ end || ::Users::Anonymous.can?(:read_project, project)
end
def http?
diff --git a/lib/gitlab/git_access_project.rb b/lib/gitlab/git_access_project.rb
index 732e0e14257..b007a957348 100644
--- a/lib/gitlab/git_access_project.rb
+++ b/lib/gitlab/git_access_project.rb
@@ -47,7 +47,7 @@ module Gitlab
end
def repository_path_match
- strong_memoize(:repository_path_match) { repository_path.match(Gitlab::PathRegex.full_project_git_path_regex) || {} }
+ strong_memoize(:repository_path_match) { repository_path&.match(Gitlab::PathRegex.full_project_git_path_regex) || {} }
end
def ensure_project_on_push!
diff --git a/lib/gitlab/git_audit_event.rb b/lib/gitlab/git_audit_event.rb
deleted file mode 100644
index b8365bdf41f..00000000000
--- a/lib/gitlab/git_audit_event.rb
+++ /dev/null
@@ -1,28 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- class GitAuditEvent # rubocop:disable Gitlab/NamespacedClass
- attr_reader :project, :user, :author
-
- def initialize(player, project)
- @project = project
- @author = player.is_a?(::API::Support::GitAccessActor) ? player.deploy_key_or_user : player
- @user = player.is_a?(::API::Support::GitAccessActor) ? player.user : player
- end
-
- def send_audit_event(msg)
- return if user.blank? || project.blank?
-
- audit_context = {
- name: 'repository_git_operation',
- stream_only: true,
- author: author,
- scope: project,
- target: project,
- message: msg
- }
-
- ::Gitlab::Audit::Auditor.audit(audit_context)
- end
- end
-end
diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb
index 5ec58fc4f44..da38c11ebca 100644
--- a/lib/gitlab/gitaly_client.rb
+++ b/lib/gitlab/gitaly_client.rb
@@ -328,6 +328,8 @@ module Gitlab
'client_name' => CLIENT_NAME
}
+ relative_path = fetch_relative_path
+
context_data = Gitlab::ApplicationContext.current
feature_stack = Thread.current[:gitaly_feature_stack]
@@ -339,6 +341,7 @@ module Gitlab
metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil)
metadata['user_id'] = context_data['meta.user_id'].to_s if context_data&.fetch('meta.user_id', nil)
metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil)
+ metadata['relative-path-bin'] = relative_path if relative_path
metadata.merge!(Feature::Gitaly.server_feature_flags(**feature_flag_actors))
metadata.merge!(route_to_primary)
@@ -348,6 +351,17 @@ module Gitlab
{ metadata: metadata, deadline: deadline_info[:deadline] }
end
+ # The GitLab `internal/allowed/` API sets the :gitlab_git_relative_path
+ # variable. This provides the repository relative path which can be used to
+ # locate snapshot repositories in Gitaly which act as a quarantine repository
+ # until a transaction is committed.
+ def self.fetch_relative_path
+ return unless Gitlab::SafeRequestStore.active?
+ return if Gitlab::SafeRequestStore[:gitlab_git_relative_path].blank?
+
+ Gitlab::SafeRequestStore.fetch(:gitlab_git_relative_path)
+ end
+
# Gitlab::Git::HookEnv will set the :gitlab_git_env variable in case we're
# running in the context of a Gitaly hook call, which may make use of
# quarantined object directories. We thus need to pass along the path of
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index 1ef5b0f96c2..3949e8e6416 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -418,6 +418,15 @@ module Gitlab
response = gitaly_client_call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout)
response.reduce([]) { |memo, msg| memo << msg.data }.join
+ rescue GRPC::BadStatus => e
+ detailed_error = GitalyClient.decode_detailed_error(e)
+
+ case detailed_error.try(:error)
+ when :out_of_range, :path_not_found
+ raise ArgumentError, e.details
+ else
+ raise e
+ end
end
def find_commit(revision)
diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
index b1278e3bfac..a6912547ce9 100644
--- a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
+++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb
@@ -43,11 +43,15 @@ module Gitlab
def conflict_from_gitaly_file_header(header)
{
- ancestor: { path: header.ancestor_path },
- ours: { path: header.our_path, mode: header.our_mode },
- theirs: { path: header.their_path }
+ ancestor: { path: encode_path(header.ancestor_path) },
+ ours: { path: encode_path(header.our_path), mode: header.our_mode },
+ theirs: { path: encode_path(header.their_path) }
}
end
+
+ def encode_path(path)
+ Gitlab::EncodingHelper.encode_utf8(path)
+ end
end
end
end
diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb
index d92bf5263f1..457380615f7 100644
--- a/lib/gitlab/gitaly_client/repository_service.rb
+++ b/lib/gitlab/gitaly_client/repository_service.rb
@@ -136,10 +136,13 @@ module Gitlab
response.base.presence
end
- def fork_repository(source_repository)
+ def fork_repository(source_repository, branch = nil)
+ revision = branch.present? ? "refs/heads/#{branch}" : ""
+
request = Gitaly::CreateForkRequest.new(
repository: @gitaly_repo,
- source_repository: source_repository.gitaly_repository
+ source_repository: source_repository.gitaly_repository,
+ revision: revision
)
gitaly_client_call(
diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb
index 4cc0269673f..adf0c811274 100644
--- a/lib/gitlab/gitaly_client/storage_settings.rb
+++ b/lib/gitlab/gitaly_client/storage_settings.rb
@@ -31,19 +31,11 @@ module Gitlab
end
def self.disk_access_denied?
- return false if rugged_enabled?
-
!temporarily_allowed?(ALLOW_KEY)
rescue StandardError
false # Err on the side of caution, don't break gitlab for people
end
- def self.rugged_enabled?
- Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.any? do |flag|
- Feature.enabled?(flag)
- end
- end
-
def initialize(storage)
raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash)
raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless storage.has_key?('path')
diff --git a/lib/gitlab/github_import/attachments_downloader.rb b/lib/gitlab/github_import/attachments_downloader.rb
index 4db55a6aabb..df9c6c8342d 100644
--- a/lib/gitlab/github_import/attachments_downloader.rb
+++ b/lib/gitlab/github_import/attachments_downloader.rb
@@ -29,8 +29,8 @@ module Gitlab
validate_content_length
validate_filepath
- redirection_url = get_assets_download_redirection_url
- file = download_from(redirection_url)
+ download_url = get_assets_download_redirection_url
+ file = download_from(download_url)
validate_symlink
file
@@ -60,16 +60,16 @@ module Gitlab
options[:follow_redirects] = false
response = Gitlab::HTTP.perform_request(Net::HTTP::Get, file_url, options)
- raise_error("expected a redirect response, got #{response.code}") unless response.redirection?
- redirection_url = response.headers[:location]
- filename = URI.parse(redirection_url).path
+ download_url = if response.redirection?
+ response.headers[:location]
+ else
+ file_url
+ end
- unless Gitlab::GithubImport::Markdown::Attachment::MEDIA_TYPES.any? { |type| filename.ends_with?(type) }
- raise UnsupportedAttachmentError
- end
+ file_type_valid?(URI.parse(download_url).path)
- redirection_url
+ download_url
end
def github_assets_url_regex
@@ -89,6 +89,12 @@ module Gitlab
File.join(dir, filename)
end
end
+
+ def file_type_valid?(file_url)
+ return if Gitlab::GithubImport::Markdown::Attachment::MEDIA_TYPES.any? { |type| file_url.ends_with?(type) }
+
+ raise UnsupportedAttachmentError
+ end
end
end
end
diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb
index 5a0ae680ab8..33e74c90115 100644
--- a/lib/gitlab/github_import/client.rb
+++ b/lib/gitlab/github_import/client.rb
@@ -182,12 +182,12 @@ module Gitlab
request_count_counter.increment
- raise_or_wait_for_rate_limit unless requests_remaining?
+ raise_or_wait_for_rate_limit('Internal threshold reached') unless requests_remaining?
begin
with_retry { yield }
- rescue ::Octokit::TooManyRequests
- raise_or_wait_for_rate_limit
+ rescue ::Octokit::TooManyRequests => e
+ raise_or_wait_for_rate_limit(e.response_body)
# This retry will only happen when running in sequential mode as we'll
# raise an error in parallel mode.
@@ -213,11 +213,11 @@ module Gitlab
octokit.rate_limit.limit
end
- def raise_or_wait_for_rate_limit
+ def raise_or_wait_for_rate_limit(message)
rate_limit_counter.increment
if parallel?
- raise RateLimitError
+ raise RateLimitError, message
else
sleep(rate_limit_resets_in)
end
diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb
index b960df581e4..0780ba6119f 100644
--- a/lib/gitlab/github_import/issuable_finder.rb
+++ b/lib/gitlab/github_import/issuable_finder.rb
@@ -11,6 +11,7 @@ module Gitlab
# The base cache key to use for storing/retrieving issuable IDs.
CACHE_KEY = 'github-import/issuable-finder/%{project}/%{type}/%{iid}'
+ CACHE_OBJECT_NOT_FOUND = -1
# project - An instance of `Project`.
# object - The object to look up or set a database ID for.
@@ -23,9 +24,18 @@ module Gitlab
#
# This method will return `nil` if no ID could be found.
def database_id
- val = Gitlab::Cache::Import::Caching.read(cache_key, timeout: timeout)
+ val = Gitlab::Cache::Import::Caching.read_integer(cache_key, timeout: timeout)
- val.to_i if val.present?
+ return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project)
+
+ return if val == CACHE_OBJECT_NOT_FOUND
+ return val if val.present?
+
+ object_id = cache_key_type.safe_constantize&.find_by(project_id: project.id, iid: cache_key_iid)&.id ||
+ CACHE_OBJECT_NOT_FOUND
+
+ cache_database_id(object_id)
+ object_id == CACHE_OBJECT_NOT_FOUND ? nil : object_id
end
# Associates the given database ID with the current object.
diff --git a/lib/gitlab/github_import/job_delay_calculator.rb b/lib/gitlab/github_import/job_delay_calculator.rb
index 52b211c92d6..077a27df16c 100644
--- a/lib/gitlab/github_import/job_delay_calculator.rb
+++ b/lib/gitlab/github_import/job_delay_calculator.rb
@@ -15,7 +15,7 @@ module Gitlab
def calculate_job_delay(job_index)
multiplier = (job_index / parallel_import_batch[:size])
- (multiplier * parallel_import_batch[:delay]) + 1.second
+ (multiplier * parallel_import_batch[:delay]).to_i + 1
end
end
end
diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb
index 39e669dbba4..d0bbd2bc7cf 100644
--- a/lib/gitlab/github_import/label_finder.rb
+++ b/lib/gitlab/github_import/label_finder.rb
@@ -7,6 +7,7 @@ module Gitlab
# The base cache key to use for storing/retrieving label IDs.
CACHE_KEY = 'github-import/label-finder/%{project}/%{name}'
+ CACHE_OBJECT_NOT_FOUND = -1
# project - An instance of `Project`.
def initialize(project)
@@ -15,7 +16,18 @@ module Gitlab
# Returns the label ID for the given name.
def id_for(name)
- Gitlab::Cache::Import::Caching.read_integer(cache_key_for(name))
+ cache_key = cache_key_for(name)
+ val = Gitlab::Cache::Import::Caching.read_integer(cache_key)
+
+ return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project)
+
+ return if val == CACHE_OBJECT_NOT_FOUND
+ return val if val.present?
+
+ object_id = project.labels.with_title(name).pick(:id) || CACHE_OBJECT_NOT_FOUND
+
+ Gitlab::Cache::Import::Caching.write(cache_key, object_id)
+ object_id == CACHE_OBJECT_NOT_FOUND ? nil : object_id
end
# rubocop: disable CodeReuse/ActiveRecord
@@ -32,7 +44,7 @@ module Gitlab
# rubocop: enable CodeReuse/ActiveRecord
def cache_key_for(name)
- CACHE_KEY % { project: project.id, name: name }
+ format(CACHE_KEY, project: project.id, name: name)
end
end
end
diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb
index d9290e36ea1..dcb679fda6d 100644
--- a/lib/gitlab/github_import/milestone_finder.rb
+++ b/lib/gitlab/github_import/milestone_finder.rb
@@ -7,6 +7,7 @@ module Gitlab
# The base cache key to use for storing/retrieving milestone IDs.
CACHE_KEY = 'github-import/milestone-finder/%{project}/%{iid}'
+ CACHE_OBJECT_NOT_FOUND = -1
# project - An instance of `Project`
def initialize(project)
@@ -18,7 +19,20 @@ module Gitlab
def id_for(issuable)
return unless issuable.milestone_number
- Gitlab::Cache::Import::Caching.read_integer(cache_key_for(issuable.milestone_number))
+ milestone_iid = issuable.milestone_number
+ cache_key = cache_key_for(milestone_iid)
+
+ val = Gitlab::Cache::Import::Caching.read_integer(cache_key)
+
+ return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project)
+
+ return if val == CACHE_OBJECT_NOT_FOUND
+ return val if val.present?
+
+ object_id = project.milestones.by_iid(milestone_iid).pick(:id) || CACHE_OBJECT_NOT_FOUND
+
+ Gitlab::Cache::Import::Caching.write(cache_key, object_id)
+ object_id == CACHE_OBJECT_NOT_FOUND ? nil : object_id
end
# rubocop: disable CodeReuse/ActiveRecord
@@ -35,7 +49,7 @@ module Gitlab
# rubocop: enable CodeReuse/ActiveRecord
def cache_key_for(iid)
- CACHE_KEY % { project: project.id, iid: iid }
+ format(CACHE_KEY, project: project.id, iid: iid)
end
end
end
diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb
index 88e91800cee..5618cfc6044 100644
--- a/lib/gitlab/github_import/object_counter.rb
+++ b/lib/gitlab/github_import/object_counter.rb
@@ -52,7 +52,7 @@ module Gitlab
.sort
.each do |counter|
object_type = counter.split('/').last
- result[operation][object_type] = CACHING.read_integer(counter) || 0
+ result[operation][object_type] = CACHING.read_integer(counter, timeout: IMPORT_CACHING_TIMEOUT) || 0
end
end
end
diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb
index cccd99f48b1..ce93b5203df 100644
--- a/lib/gitlab/github_import/parallel_scheduling.rb
+++ b/lib/gitlab/github_import/parallel_scheduling.rb
@@ -6,7 +6,7 @@ module Gitlab
include JobDelayCalculator
attr_reader :project, :client, :page_counter, :already_imported_cache_key,
- :job_waiter_cache_key, :job_waiter_remaining_cache_key
+ :job_waiter_cache_key, :job_waiter_remaining_cache_key
# The base cache key to use for tracking already imported objects.
ALREADY_IMPORTED_CACHE_KEY =
@@ -26,12 +26,11 @@ module Gitlab
@client = client
@parallel = parallel
@page_counter = PageCounter.new(project, collection_method)
- @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY %
- { project: project.id, collection: collection_method }
- @job_waiter_cache_key = JOB_WAITER_CACHE_KEY %
- { project: project.id, collection: collection_method }
- @job_waiter_remaining_cache_key = JOB_WAITER_REMAINING_CACHE_KEY %
- { project: project.id, collection: collection_method }
+ @already_imported_cache_key = format(ALREADY_IMPORTED_CACHE_KEY, project: project.id,
+ collection: collection_method)
+ @job_waiter_cache_key = format(JOB_WAITER_CACHE_KEY, project: project.id, collection: collection_method)
+ @job_waiter_remaining_cache_key = format(JOB_WAITER_REMAINING_CACHE_KEY, project: project.id,
+ collection: collection_method)
end
def parallel?
@@ -57,7 +56,8 @@ module Gitlab
# still scheduling duplicates while. Since all work has already been
# completed those jobs will just cycle through any remaining pages while
# not scheduling anything.
- Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT)
+ Gitlab::Cache::Import::Caching.expire(already_imported_cache_key,
+ Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT)
info(project.id, message: "importer finished")
retval
@@ -97,7 +97,7 @@ module Gitlab
repr = object_representation(object)
job_delay = calculate_job_delay(enqueued_job_counter)
- sidekiq_worker_class.perform_in(job_delay, project.id, repr.to_hash, job_waiter.key)
+ sidekiq_worker_class.perform_in(job_delay, project.id, repr.to_hash.deep_stringify_keys, job_waiter.key.to_s)
enqueued_job_counter += 1
job_waiter.jobs_remaining = Gitlab::Cache::Import::Caching.increment(job_waiter_remaining_cache_key)
diff --git a/lib/gitlab/github_import/representation/to_hash.rb b/lib/gitlab/github_import/representation/to_hash.rb
index 4a0f36ab8f0..54faa51a787 100644
--- a/lib/gitlab/github_import/representation/to_hash.rb
+++ b/lib/gitlab/github_import/representation/to_hash.rb
@@ -16,11 +16,15 @@ module Gitlab
hash
end
+ # This method allow objects to be safely passed directly to Sidekiq without errors.
+ # It returns JSON datatypes: string, integer, float, boolean, null(nil), array and hash.
def convert_value_for_to_hash(value)
if value.is_a?(Array)
value.map { |v| convert_value_for_to_hash(v) }
elsif value.respond_to?(:to_hash)
value.to_hash
+ elsif value.respond_to?(:strftime) || value.is_a?(Symbol)
+ value.to_s
else
value
end
diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb
index e057b4bb6f1..59813e4f5a0 100644
--- a/lib/gitlab/gon_helper.rb
+++ b/lib/gitlab/gon_helper.rb
@@ -50,6 +50,7 @@ module Gitlab
gon.suggested_label_colors = LabelsHelper.suggested_colors
gon.first_day_of_week = current_user&.first_day_of_week || Gitlab::CurrentSettings.first_day_of_week
gon.time_display_relative = true
+ gon.time_display_format = 0
gon.ee = Gitlab.ee?
gon.jh = Gitlab.jh?
gon.dot_com = Gitlab.com?
@@ -67,6 +68,7 @@ module Gitlab
gon.current_user_fullname = current_user.name
gon.current_user_avatar_url = current_user.avatar_url
gon.time_display_relative = current_user.time_display_relative
+ gon.time_display_format = current_user.time_display_format
end
# Initialize gon.features with any flags that should be
@@ -75,7 +77,6 @@ module Gitlab
push_frontend_feature_flag(:security_auto_fix)
push_frontend_feature_flag(:source_editor_toolbar)
push_frontend_feature_flag(:vscode_web_ide, current_user)
- push_frontend_feature_flag(:unbatch_graphql_queries, current_user)
# To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248
push_frontend_feature_flag(:remove_monitor_metrics)
push_frontend_feature_flag(:custom_emoji)
diff --git a/lib/gitlab/graphql/tracers/timer_tracer.rb b/lib/gitlab/graphql/tracers/timer_tracer.rb
index 8e058621110..2cf06086a3c 100644
--- a/lib/gitlab/graphql/tracers/timer_tracer.rb
+++ b/lib/gitlab/graphql/tracers/timer_tracer.rb
@@ -15,11 +15,11 @@ module Gitlab
end
def trace(key, data)
- start_time = Gitlab::Metrics::System.monotonic_time
+ start_time = ::Gitlab::Metrics::System.monotonic_time
yield
ensure
- data[:duration_s] = Gitlab::Metrics::System.monotonic_time - start_time
+ data[:duration_s] = ::Gitlab::Metrics::System.monotonic_time - start_time
end
end
end
diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb
index 8ca88859b22..6fe7a0030f0 100644
--- a/lib/gitlab/group_search_results.rb
+++ b/lib/gitlab/group_search_results.rb
@@ -13,7 +13,7 @@ module Gitlab
# rubocop:disable CodeReuse/ActiveRecord
def users
groups = group.self_and_hierarchy_intersecting_with_user_groups(current_user)
- groups = groups.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/417455")
+ groups = groups.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/427108")
members = GroupMember.where(group: groups).non_invite
users = super
diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb
index 08d44184bb6..720f8748cba 100644
--- a/lib/gitlab/identifier.rb
+++ b/lib/gitlab/identifier.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-# Detect user based on identifier like
+# Detect user or keys based on identifier like
# key-13 or user-36
module Gitlab
module Identifier
@@ -35,6 +35,13 @@ module Gitlab
end
end
+ # Tries to identify a deploy key using a SSH key identifier (e.g. "key-123").
+ def identify_using_deploy_key(identifier)
+ key_id = identifier.gsub("key-", "")
+
+ DeployKey.find_by_id(key_id)
+ end
+
def identify_with_cache(category, key)
if identification_cache[category].key?(key)
identification_cache[category][key]
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index ea91b01afdb..523df1f9d5e 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -40,13 +40,12 @@ module Gitlab
cmd = %W[gzip #{filepath}]
cmd << "-#{options}" if options
- _, status = Gitlab::Popen.popen(cmd)
+ output, status = Gitlab::Popen.popen(cmd)
- if status == 0
- status
- else
- raise Gitlab::ImportExport::Error.file_compression_error
- end
+ return status if status == 0
+
+ message = cmd_error_message(output, status)
+ raise Gitlab::ImportExport::Error.file_compression_error(message)
end
def mkdir_p(path)
@@ -104,9 +103,7 @@ module Gitlab
return true if status == 0
- output = output&.strip
- message = "command exited with error code #{status}"
- message += ": #{output}" if output.present?
+ message = cmd_error_message(output, status)
if @shared.respond_to?(:error)
@shared.error(Gitlab::ImportExport::Error.new(message))
@@ -149,6 +146,12 @@ module Gitlab
FileUtils.remove_dir(dir)
raise
end
+
+ def cmd_error_message(output, status)
+ message = "Command exited with error code #{status}"
+ message << ": #{output.strip}" unless output.blank?
+ message
+ end
end
end
end
diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb
index fa179f584eb..9b8e6374b5a 100644
--- a/lib/gitlab/import_export/error.rb
+++ b/lib/gitlab/import_export/error.rb
@@ -14,8 +14,8 @@ module Gitlab
self.new('Unknown object type')
end
- def self.file_compression_error
- self.new('File compression/decompression failed')
+ def self.file_compression_error(error)
+ self.new(format('File compression or decompression failed. %{error}', error: error))
end
def self.incompatible_import_file_error
diff --git a/lib/gitlab/import_export/project/sample/date_calculator.rb b/lib/gitlab/import_export/project/sample/date_calculator.rb
index 543fd25d883..0cb0eb32a23 100644
--- a/lib/gitlab/import_export/project/sample/date_calculator.rb
+++ b/lib/gitlab/import_export/project/sample/date_calculator.rb
@@ -25,7 +25,7 @@ module Gitlab
end
def calculate_by_closest_date_to_average(date)
- return date unless closest_date_to_average && closest_date_to_average < Time.current
+ return date unless closest_date_to_average && closest_date_to_average.past?
date + (Time.current - closest_date_to_average).seconds
end
diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb
index e39bbb36680..88991495a10 100644
--- a/lib/gitlab/instrumentation/redis_base.rb
+++ b/lib/gitlab/instrumentation/redis_base.rb
@@ -90,7 +90,7 @@ module Gitlab
result = ::Gitlab::Instrumentation::RedisClusterValidator.validate(commands)
return true if result.nil?
- if !result[:valid] && !result[:allowed] && (Rails.env.development? || Rails.env.test?)
+ if !result[:valid] && !result[:allowed] && raise_cross_slot_validation_errors?
raise RedisClusterValidator::CrossSlotError, "Redis command #{result[:command_name]} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands"
end
@@ -189,6 +189,10 @@ module Gitlab
redirection_type, _, target_node_key = err_msg.split
{ redirection_type: redirection_type, target_node_key: target_node_key }
end
+
+ def raise_cross_slot_validation_errors?
+ Rails.env.development? || Rails.env.test?
+ end
end
end
end
diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb
index 20ba1ab82a7..5934204bd0f 100644
--- a/lib/gitlab/instrumentation/redis_interceptor.rb
+++ b/lib/gitlab/instrumentation/redis_interceptor.rb
@@ -31,7 +31,7 @@ module Gitlab
private
def instrument_call(commands, pipelined = false)
- start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined
+ start = ::Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined
instrumentation_class.instance_count_request(commands.size)
instrumentation_class.instance_count_pipelined_request(commands.size) if pipelined
@@ -50,7 +50,7 @@ module Gitlab
instrumentation_class.log_exception(ex)
raise ex
ensure
- duration = Gitlab::Metrics::System.monotonic_time - start
+ duration = ::Gitlab::Metrics::System.monotonic_time - start
unless exclude_from_apdex?(commands)
commands.each { instrumentation_class.instance_observe_duration(duration / commands.size) }
diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb
index 2a3c4db5ffa..49078a7ccd0 100644
--- a/lib/gitlab/instrumentation_helper.rb
+++ b/lib/gitlab/instrumentation_helper.rb
@@ -12,7 +12,6 @@ module Gitlab
def add_instrumentation_data(payload)
instrument_gitaly(payload)
- instrument_rugged(payload)
instrument_redis(payload)
instrument_elasticsearch(payload)
instrument_zoekt(payload)
@@ -40,15 +39,6 @@ module Gitlab
payload[:gitaly_duration_s] = Gitlab::GitalyClient.query_time
end
- def instrument_rugged(payload)
- rugged_calls = Gitlab::RuggedInstrumentation.query_count
-
- return if rugged_calls == 0
-
- payload[:rugged_calls] = rugged_calls
- payload[:rugged_duration_s] = Gitlab::RuggedInstrumentation.query_time
- end
-
def instrument_redis(payload)
payload.merge! ::Gitlab::Instrumentation::Redis.payload
end
diff --git a/lib/gitlab/internal_events.rb b/lib/gitlab/internal_events.rb
index 2790bc8ee24..e2e4ea75dbf 100644
--- a/lib/gitlab/internal_events.rb
+++ b/lib/gitlab/internal_events.rb
@@ -23,8 +23,6 @@ module Gitlab
private
def increase_total_counter(event_name)
- return unless ::ServicePing::ServicePingSettings.enabled?
-
redis_counter_key =
Gitlab::Usage::Metrics::Instrumentations::TotalCountMetric.redis_key(event_name)
Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) }
diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb
index 12cc5f6e5dd..c60dac6f571 100644
--- a/lib/gitlab/issues/rebalancing/state.rb
+++ b/lib/gitlab/issues/rebalancing/state.rb
@@ -100,7 +100,7 @@ module Gitlab
def refresh_keys_expiration
with_redis do |redis|
Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
- redis.pipelined do |pipeline|
+ Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline|
pipeline.expire(issue_ids_key, REDIS_EXPIRY_TIME)
pipeline.expire(current_index_key, REDIS_EXPIRY_TIME)
pipeline.expire(current_project_key, REDIS_EXPIRY_TIME)
diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb
index 7abfe8e38e8..2b8b01e2023 100644
--- a/lib/gitlab/jira/http_client.rb
+++ b/lib/gitlab/jira/http_client.rb
@@ -34,6 +34,17 @@ module Gitlab
request_params[:headers][:Cookie] = get_cookies if options[:use_cookies]
request_params[:base_uri] = uri.to_s
request_params.merge!(auth_params)
+ # Setting defaults here so we can also set `timeout` which prevents setting defaults in the HTTP gem's code
+ request_params[:open_timeout] = options[:open_timeout] || default_timeout_for(:open_timeout)
+ request_params[:read_timeout] = options[:read_timeout] || default_timeout_for(:read_timeout)
+ request_params[:write_timeout] = options[:write_timeout] || default_timeout_for(:write_timeout)
+ # Global timeout. Needs to be at least as high as the maximum defined in other timeouts
+ request_params[:timeout] = [
+ Gitlab::HTTP::DEFAULT_READ_TOTAL_TIMEOUT,
+ request_params[:open_timeout],
+ request_params[:read_timeout],
+ request_params[:write_timeout]
+ ].max
result = Gitlab::HTTP.public_send(http_method, path, **request_params) # rubocop:disable GitlabSecurity/PublicSend
@authenticated = result.response.is_a?(Net::HTTPOK)
@@ -52,6 +63,10 @@ module Gitlab
private
+ def default_timeout_for(param)
+ Gitlab::HTTP::DEFAULT_TIMEOUT_OPTIONS[param]
+ end
+
def auth_params
return {} unless @options[:username] && @options[:password]
diff --git a/lib/gitlab/jira/middleware.rb b/lib/gitlab/jira/middleware.rb
deleted file mode 100644
index 8a74729da49..00000000000
--- a/lib/gitlab/jira/middleware.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Jira
- class Middleware
- def self.jira_dvcs_connector?(env)
- env['HTTP_USER_AGENT']&.downcase&.start_with?('jira dvcs connector')
- end
-
- def initialize(app)
- @app = app
- end
-
- def call(env)
- if self.class.jira_dvcs_connector?(env)
- env['HTTP_AUTHORIZATION'] = env['HTTP_AUTHORIZATION']&.sub('token', 'Bearer')
- end
-
- @app.call(env)
- end
- end
- end
-end
diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb
index 2b83f0492cb..04ef1a0ef68 100644
--- a/lib/gitlab/jira_import/base_importer.rb
+++ b/lib/gitlab/jira_import/base_importer.rb
@@ -5,7 +5,7 @@ module Gitlab
class BaseImporter
attr_reader :project, :client, :formatter, :jira_project_key, :running_import
- def initialize(project)
+ def initialize(project, client = nil)
Gitlab::JiraImport.validate_project_settings!(project)
@running_import = project.latest_jira_import
@@ -14,7 +14,7 @@ module Gitlab
raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key
@project = project
- @client = project.jira_integration.client
+ @client = client || project.jira_integration.client
@formatter = Gitlab::ImportFormatter.new
end
diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb
index 458f7c3f470..54ececc4938 100644
--- a/lib/gitlab/jira_import/issues_importer.rb
+++ b/lib/gitlab/jira_import/issues_importer.rb
@@ -10,7 +10,7 @@ module Gitlab
attr_reader :imported_items_cache_key, :start_at, :job_waiter
- def initialize(project)
+ def initialize(project, client = nil)
super
# get cached start_at value, or zero if not cached yet
@start_at = Gitlab::JiraImport.get_issues_next_start_at(project.id)
diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb
index e53bfb40654..7b491b3e14d 100644
--- a/lib/gitlab/job_waiter.rb
+++ b/lib/gitlab/job_waiter.rb
@@ -19,9 +19,6 @@ module Gitlab
class JobWaiter
KEY_PREFIX = "gitlab:job_waiter"
- STARTED_METRIC = :gitlab_job_waiter_started_total
- TIMEOUTS_METRIC = :gitlab_job_waiter_timeouts_total
-
# This TTL needs to be long enough to allow whichever Sidekiq job calls
# JobWaiter#wait to reach BLPOP.
DEFAULT_TTL = 6.hours.to_i
@@ -48,16 +45,15 @@ module Gitlab
Gitlab::Redis::SharedState.with { |redis| redis.del(key) } if key?(key)
end
- attr_reader :key, :finished, :worker_label
+ attr_reader :key, :finished
attr_accessor :jobs_remaining
# jobs_remaining - the number of jobs left to wait for
# key - The key of this waiter.
- def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}", worker_label: nil)
+ def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}")
@key = key
@jobs_remaining = jobs_remaining
@finished = []
- @worker_label = worker_label
end
# Waits for all the jobs to be completed.
@@ -67,7 +63,6 @@ module Gitlab
# long to process, or is never processed.
def wait(timeout = 10)
deadline = Time.now.utc + timeout
- increment_counter(STARTED_METRIC)
Gitlab::Redis::SharedState.with do |redis|
while jobs_remaining > 0
@@ -81,10 +76,7 @@ module Gitlab
list, jid = redis.blpop(key, timeout: seconds_left)
# timed out
- unless list && jid
- increment_counter(TIMEOUTS_METRIC)
- break
- end
+ break unless list && jid
@finished << jid
@jobs_remaining -= 1
@@ -93,20 +85,5 @@ module Gitlab
finished
end
-
- private
-
- def increment_counter(metric)
- return unless worker_label
-
- metrics[metric].increment(worker: worker_label)
- end
-
- def metrics
- @metrics ||= {
- STARTED_METRIC => Gitlab::Metrics.counter(STARTED_METRIC, 'JobWaiter attempts started'),
- TIMEOUTS_METRIC => Gitlab::Metrics.counter(TIMEOUTS_METRIC, 'JobWaiter attempts timed out')
- }
- end
end
end
diff --git a/lib/gitlab/kubernetes/kubeconfig/template.rb b/lib/gitlab/kubernetes/kubeconfig/template.rb
index d40b9ce117e..844472f9c8e 100644
--- a/lib/gitlab/kubernetes/kubeconfig/template.rb
+++ b/lib/gitlab/kubernetes/kubeconfig/template.rb
@@ -44,7 +44,7 @@ module Gitlab
)
end
kubeconfig_yaml[:clusters].each do |cluster|
- ca_pem = cluster.dig(:cluster, :'certificate-authority-data')&.yield_self do |data|
+ ca_pem = cluster.dig(:cluster, :'certificate-authority-data')&.then do |data|
Base64.strict_decode64(data)
end
diff --git a/lib/gitlab/legacy_http.rb b/lib/gitlab/legacy_http.rb
index f38b2819c15..cf6ab80d37f 100644
--- a/lib/gitlab/legacy_http.rb
+++ b/lib/gitlab/legacy_http.rb
@@ -35,8 +35,8 @@ module Gitlab
read_total_timeout = options.fetch(:timeout, Gitlab::HTTP::DEFAULT_READ_TOTAL_TIMEOUT)
httparty_perform_request(http_method, path, options_with_timeouts) do |fragment|
- start_time ||= Gitlab::Metrics::System.monotonic_time
- elapsed = Gitlab::Metrics::System.monotonic_time - start_time
+ start_time ||= ::Gitlab::Metrics::System.monotonic_time
+ elapsed = ::Gitlab::Metrics::System.monotonic_time - start_time
if elapsed > read_total_timeout
raise Gitlab::HTTP::ReadTotalTimeout, "Request timed out after #{elapsed} seconds"
diff --git a/lib/gitlab/memory/reporter.rb b/lib/gitlab/memory/reporter.rb
index db0fd24983b..8d32745ac34 100644
--- a/lib/gitlab/memory/reporter.rb
+++ b/lib/gitlab/memory/reporter.rb
@@ -26,13 +26,13 @@ module Gitlab
perf_report: report.name
))
- start_monotonic_time = Gitlab::Metrics::System.monotonic_time
- start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time
+ start_monotonic_time = ::Gitlab::Metrics::System.monotonic_time
+ start_thread_cpu_time = ::Gitlab::Metrics::System.thread_cpu_time
report_file = store_report(report)
- cpu_s = Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time)
- duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time
+ cpu_s = ::Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time)
+ duration_s = ::Gitlab::Metrics::System.monotonic_time - start_monotonic_time
@logger.info(
log_labels(
diff --git a/lib/gitlab/memory/reports_uploader.rb b/lib/gitlab/memory/reports_uploader.rb
index 76c3e0862e2..17230414a6a 100644
--- a/lib/gitlab/memory/reports_uploader.rb
+++ b/lib/gitlab/memory/reports_uploader.rb
@@ -13,11 +13,11 @@ module Gitlab
def upload(path)
log_upload_requested(path)
- start_monotonic_time = Gitlab::Metrics::System.monotonic_time
+ start_monotonic_time = ::Gitlab::Metrics::System.monotonic_time
File.open(path.to_s) { |file| fog.put_object(gcs_bucket, File.basename(path), file) }
- duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time
+ duration_s = ::Gitlab::Metrics::System.monotonic_time - start_monotonic_time
log_upload_success(path, duration_s)
rescue StandardError, Errno::ENOENT => error
log_exception(error)
diff --git a/lib/gitlab/merge_requests/mergeability/check_result.rb b/lib/gitlab/merge_requests/mergeability/check_result.rb
index e18909d8f17..075a897478b 100644
--- a/lib/gitlab/merge_requests/mergeability/check_result.rb
+++ b/lib/gitlab/merge_requests/mergeability/check_result.rb
@@ -5,6 +5,7 @@ module Gitlab
class CheckResult
SUCCESS_STATUS = :success
FAILED_STATUS = :failed
+ INACTIVE_STATUS = :inactive
attr_reader :status, :payload
@@ -20,6 +21,10 @@ module Gitlab
new(status: FAILED_STATUS, payload: default_payload.merge(**payload))
end
+ def self.inactive(payload: {})
+ new(status: INACTIVE_STATUS, payload: default_payload.merge(**payload))
+ end
+
def self.from_hash(data)
new(
status: data.fetch(:status).to_sym,
diff --git a/lib/gitlab/metrics/exporter/metrics_middleware.rb b/lib/gitlab/metrics/exporter/metrics_middleware.rb
index 258b655229e..b80a8c503e8 100644
--- a/lib/gitlab/metrics/exporter/metrics_middleware.rb
+++ b/lib/gitlab/metrics/exporter/metrics_middleware.rb
@@ -9,10 +9,10 @@ module Gitlab
default_labels = {
pid: pid
}
- @requests_total = Gitlab::Metrics.counter(
+ @requests_total = ::Gitlab::Metrics.counter(
:exporter_http_requests_total, 'Total number of HTTP requests', default_labels
)
- @request_durations = Gitlab::Metrics.histogram(
+ @request_durations = ::Gitlab::Metrics.histogram(
:exporter_http_request_duration_seconds,
'HTTP request duration histogram (seconds)',
default_labels,
@@ -21,9 +21,9 @@ module Gitlab
end
def call(env)
- start = Gitlab::Metrics::System.monotonic_time
+ start = ::Gitlab::Metrics::System.monotonic_time
@app.call(env).tap do |response|
- duration = Gitlab::Metrics::System.monotonic_time - start
+ duration = ::Gitlab::Metrics::System.monotonic_time - start
labels = {
method: env['REQUEST_METHOD'].downcase,
diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb
index d2336ec4bb2..5a0612be88e 100644
--- a/lib/gitlab/middleware/go.rb
+++ b/lib/gitlab/middleware/go.rb
@@ -141,7 +141,7 @@ module Gitlab
return empty_result unless has_basic_credentials?(request)
login, password = user_name_and_password(request)
- auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, ip: request.ip)
+ auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, request: request)
return empty_result unless auth_result.success?
return empty_result unless auth_result.can?(:access_git)
diff --git a/lib/gitlab/middleware/path_traversal_check.rb b/lib/gitlab/middleware/path_traversal_check.rb
index 79465f3cb30..6fef247b708 100644
--- a/lib/gitlab/middleware/path_traversal_check.rb
+++ b/lib/gitlab/middleware/path_traversal_check.rb
@@ -5,6 +5,28 @@ module Gitlab
class PathTraversalCheck
PATH_TRAVERSAL_MESSAGE = 'Potential path traversal attempt detected'
+ EXCLUDED_EXACT_PATHS = %w[/search].freeze
+ EXCLUDED_PATH_PREFIXES = %w[/search/].freeze
+
+ EXCLUDED_API_PATHS = %w[/search].freeze
+ EXCLUDED_PROJECT_API_PATHS = %w[/search].freeze
+ EXCLUDED_GROUP_API_PATHS = %w[/search].freeze
+
+ API_PREFIX = %r{/api/[^/]+}
+ API_SUFFIX = %r{(?:\.[^/]+)?}
+
+ EXCLUDED_API_PATHS_REGEX = [
+ EXCLUDED_API_PATHS.map do |path|
+ %r{\A#{API_PREFIX}#{path}#{API_SUFFIX}\z}
+ end.freeze,
+ EXCLUDED_PROJECT_API_PATHS.map do |path|
+ %r{\A#{API_PREFIX}/projects/[^/]+(?:/-)?#{path}#{API_SUFFIX}\z}
+ end.freeze,
+ EXCLUDED_GROUP_API_PATHS.map do |path|
+ %r{\A#{API_PREFIX}/groups/[^/]+(?:/-)?#{path}#{API_SUFFIX}\z}
+ end.freeze
+ ].flatten.freeze
+
def initialize(app)
@app = app
end
@@ -14,7 +36,8 @@ module Gitlab
log_params = {}
execution_time = measure_execution_time do
- check(env, log_params)
+ request = ::Rack::Request.new(env.dup)
+ check(request, log_params) unless excluded?(request)
end
log_params[:duration_ms] = execution_time.round(5) if execution_time
@@ -37,17 +60,25 @@ module Gitlab
end
end
- def check(env, log_params)
- request = ::Rack::Request.new(env)
- fullpath = request.fullpath
- decoded_fullpath = CGI.unescape(fullpath)
+ def check(request, log_params)
+ decoded_fullpath = CGI.unescape(request.fullpath)
::Gitlab::PathTraversal.check_path_traversal!(decoded_fullpath, skip_decoding: true)
-
rescue ::Gitlab::PathTraversal::PathTraversalAttackError
- log_params[:fullpath] = fullpath
+ log_params[:method] = request.request_method
+ log_params[:fullpath] = request.fullpath
log_params[:message] = PATH_TRAVERSAL_MESSAGE
end
+ def excluded?(request)
+ path = request.path
+
+ return true if path.in?(EXCLUDED_EXACT_PATHS)
+ return true if EXCLUDED_PATH_PREFIXES.any? { |p| path.start_with?(p) }
+ return true if EXCLUDED_API_PATHS_REGEX.any? { |r| path.match?(r) }
+
+ false
+ end
+
def log(payload)
Gitlab::AppLogger.warn(
payload.merge(class_name: self.class.name)
diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb
index 81ad7a7f9e1..0bcd5b1196a 100644
--- a/lib/gitlab/omniauth_initializer.rb
+++ b/lib/gitlab/omniauth_initializer.rb
@@ -29,6 +29,8 @@ module Gitlab
{
authorize_params: { gl_auth_type: 'login' }
}
+ when ->(provider_name) { AuthHelper.saml_providers.include?(provider_name.to_sym) }
+ { attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements }
else
{}
end
@@ -61,7 +63,7 @@ module Gitlab
provider_arguments.concat arguments
provider_arguments << defaults unless defaults.empty?
when Hash, GitlabSettings::Options
- hash_arguments = arguments.deep_symbolize_keys.deep_merge(defaults)
+ hash_arguments = merge_hash_defaults_and_args(defaults, arguments)
normalized = normalize_hash_arguments(hash_arguments)
# A Hash from the configuration will be passed as is.
@@ -80,6 +82,13 @@ module Gitlab
provider_arguments
end
+ def merge_hash_defaults_and_args(defaults, arguments)
+ return arguments.to_hash if defaults.empty?
+ return defaults.deep_merge(arguments.deep_symbolize_keys) if Feature.enabled?(:invert_omniauth_args_merging)
+
+ arguments.to_hash.deep_symbolize_keys.deep_merge(defaults)
+ end
+
def normalize_hash_arguments(args)
args.deep_symbolize_keys!
diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb
index 3c8ac55f70b..adc417f287c 100644
--- a/lib/gitlab/optimistic_locking.rb
+++ b/lib/gitlab/optimistic_locking.rb
@@ -7,7 +7,7 @@ module Gitlab
module_function
def retry_lock(subject, max_retries = MAX_RETRIES, name:, &block)
- start_time = Gitlab::Metrics::System.monotonic_time
+ start_time = ::Gitlab::Metrics::System.monotonic_time
retry_attempts = 0
# prevent scope override, see https://gitlab.com/gitlab-org/gitlab/-/issues/391186
@@ -39,7 +39,7 @@ module Gitlab
def log_optimistic_lock_retries(name:, retry_attempts:, start_time:)
return unless retry_attempts > 0
- elapsed_time = Gitlab::Metrics::System.monotonic_time - start_time
+ elapsed_time = ::Gitlab::Metrics::System.monotonic_time - start_time
retry_lock_logger.info(
message: "Optimistic Lock released with retries",
diff --git a/lib/gitlab/pages/deployment_update.rb b/lib/gitlab/pages/deployment_update.rb
index 6845f5d88ec..bf6ac3a056d 100644
--- a/lib/gitlab/pages/deployment_update.rb
+++ b/lib/gitlab/pages/deployment_update.rb
@@ -89,14 +89,10 @@ module Gitlab
project.actual_limits.pages_file_entries
end
+ # If a newer pipeline already build a PagesDeployment
def validate_outdated_sha
return if latest?
-
- # use pipeline_id in case the build is retried
- last_deployed_pipeline_id = project.pages_metadatum&.pages_deployment&.ci_build&.pipeline_id
-
- return unless last_deployed_pipeline_id
- return if last_deployed_pipeline_id <= build.pipeline_id
+ return if latest_pipeline_id <= build.pipeline_id
errors.add(:base, 'build SHA is outdated for this ref')
end
@@ -111,6 +107,13 @@ module Gitlab
def sha
build.sha
end
+
+ def latest_pipeline_id
+ project
+ .active_pages_deployments
+ .with_path_prefix(build.pages&.dig(:path_prefix))
+ .latest_pipeline_id
+ end
end
end
end
diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb
index 81dcc54ff35..9e8c0c530a9 100644
--- a/lib/gitlab/pagination/cursor_based_keyset.rb
+++ b/lib/gitlab/pagination/cursor_based_keyset.rb
@@ -34,8 +34,10 @@ module Gitlab
order_satisfied?(relation, cursor_based_request_context)
end
- def self.enforced_for_type?(relation)
- ENFORCED_TYPES.include?(relation.klass)
+ def self.enforced_for_type?(request_scope, relation)
+ enforced = ENFORCED_TYPES
+ enforced += [::Ci::Build] if ::Feature.enabled?(:enforce_ci_builds_pagination_limit, request_scope, type: :ops)
+ enforced.include?(relation.klass)
end
def self.order_satisfied?(relation, cursor_based_request_context)
diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb
index c9eae2f899f..8f1fbf53161 100644
--- a/lib/gitlab/patch/sidekiq_cron_poller.rb
+++ b/lib/gitlab/patch/sidekiq_cron_poller.rb
@@ -7,7 +7,7 @@
require 'sidekiq/version'
require 'sidekiq/cron/version'
-if Gem::Version.new(Sidekiq::VERSION) != Gem::Version.new('6.5.7')
+if Gem::Version.new(Sidekiq::VERSION) != Gem::Version.new('6.5.12')
raise 'New version of sidekiq detected, please remove or update this patch'
end
diff --git a/lib/gitlab/patch/sidekiq_scheduled_enq.rb b/lib/gitlab/patch/sidekiq_scheduled_enq.rb
index de0e8465f97..b5a40c19923 100644
--- a/lib/gitlab/patch/sidekiq_scheduled_enq.rb
+++ b/lib/gitlab/patch/sidekiq_scheduled_enq.rb
@@ -15,10 +15,8 @@ module Gitlab
# this portion swaps out Sidekiq.redis for Gitlab::Redis::Queues
Gitlab::Redis::Queues.with do |conn| # rubocop:disable Cop/RedisQueueUsage
sorted_sets.each do |sorted_set|
- # adds namespace if `super` polls with a non-namespaced Sidekiq.redis
- if Gitlab::Utils.to_boolean(ENV['SIDEKIQ_ENQUEUE_NON_NAMESPACED'])
- sorted_set = "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{sorted_set}" # rubocop:disable Cop/RedisQueueUsage
- end
+ # adds namespace since `super` polls with a non-namespaced Sidekiq.redis
+ sorted_set = "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{sorted_set}" # rubocop:disable Cop/RedisQueueUsage
while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s])) # rubocop:disable Gitlab/ModuleWithInstanceVariables, Lint/AssignmentInCondition
Sidekiq::Client.push(Sidekiq.load_json(job)) # rubocop:disable Cop/SidekiqApiUsage
@@ -28,7 +26,6 @@ module Gitlab
end
end
- # calls original enqueue_jobs which may or may not be namespaced depending on SIDEKIQ_ENQUEUE_NON_NAMESPACED
super
end
end
diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb
index c9ed4720e83..5f2084ce011 100644
--- a/lib/gitlab/project_template.rb
+++ b/lib/gitlab/project_template.rb
@@ -60,14 +60,14 @@ module Gitlab
ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/third-party-logos/dotnet.svg'),
ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'),
ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'),
- ProjectTemplate.new('bridgetown', 'Pages/Bridgetown', _('Everything you need to create a GitLab Pages site using Bridgetown'), 'https://gitlab.com/gitlab-org/project-templates/bridgetown'),
+ ProjectTemplate.new('bridgetown', 'Pages/Bridgetown', _('Everything you need to create a GitLab Pages site using Bridgetown'), 'https://gitlab.com/pages/bridgetown'),
ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby'), 'https://gitlab.com/pages/gatsby', 'illustrations/third-party-logos/gatsby.svg'),
ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'),
ProjectTemplate.new('pelican', 'Pages/Pelican', _('Everything you need to create a GitLab Pages site using Pelican'), 'https://gitlab.com/pages/pelican', 'illustrations/third-party-logos/pelican.svg'),
ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll'), 'https://gitlab.com/pages/jekyll', 'illustrations/logos/jekyll.svg'),
ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML'), 'https://gitlab.com/pages/plain-html'),
ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'),
- ProjectTemplate.new('middleman', 'Pages/Middleman', _('Everything you need to create a GitLab Pages site using Middleman'), 'https://gitlab.com/gitlab-org/project-templates/middleman', 'illustrations/logos/middleman.svg'),
+ ProjectTemplate.new('middleman', 'Pages/Middleman', _('Everything you need to create a GitLab Pages site using Middleman'), 'https://gitlab.com/pages/middleman', 'illustrations/logos/middleman.svg'),
ProjectTemplate.new('gitpod_spring_petclinic', 'Gitpod/Spring Petclinic', _('A Gitpod configured Webapplication in Spring and Java'), 'https://gitlab.com/gitlab-org/project-templates/gitpod-spring-petclinic', 'illustrations/logos/gitpod.svg'),
ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'),
ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'),
@@ -81,7 +81,8 @@ module Gitlab
ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'),
ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux'),
ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/gitlab-org/project-templates/typo3-distribution', 'illustrations/logos/typo3.svg'),
- ProjectTemplate.new('laravel', 'Laravel Framework', _('A basic folder structure of a Laravel application, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/laravel', 'illustrations/logos/laravel.svg')
+ ProjectTemplate.new('laravel', 'Laravel Framework', _('A basic folder structure of a Laravel application, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/laravel', 'illustrations/logos/laravel.svg'),
+ ProjectTemplate.new('astro_tailwind', 'Astro Tailwind', _('A basic folder structure of Astro Starter Kit, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/astro-tailwind')
]
end
# rubocop:enable Metrics/AbcSize
diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb
index 4471d21b9ac..e817f2130f4 100644
--- a/lib/gitlab/push_options.rb
+++ b/lib/gitlab/push_options.rb
@@ -14,6 +14,7 @@ module Gitlab
:milestone,
:remove_source_branch,
:target,
+ :target_project,
:title,
:unassign,
:unlabel
diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb
index 9798b0eca2c..72bec159226 100644
--- a/lib/gitlab/quick_actions/merge_request_actions.rb
+++ b/lib/gitlab/quick_actions/merge_request_actions.rb
@@ -172,6 +172,25 @@ module Gitlab
end
end
+ desc { _('Request changes') }
+ explanation { _('Request changes to the current merge request.') }
+ types MergeRequest
+ condition do
+ Feature.enabled?(:mr_request_changes, current_user) &&
+ quick_action_target.persisted? &&
+ quick_action_target.find_reviewer(current_user)
+ end
+ command :request_changes do
+ result = ::MergeRequests::UpdateReviewerStateService.new(project: quick_action_target.project, current_user: current_user)
+ .execute(quick_action_target, "requested_changes")
+
+ @execution_message[:request_changes] = if result[:status] == :success
+ _('Changes requested to the current merge request.')
+ else
+ result[:message]
+ end
+ end
+
desc { _('Approve a merge request') }
explanation { _('Approve the current merge request.') }
types MergeRequest
@@ -197,6 +216,10 @@ module Gitlab
next unless success
+ ::MergeRequests::UpdateReviewerStateService
+ .new(project: quick_action_target.project, current_user: current_user)
+ .execute(quick_action_target, "unreviewed")
+
@execution_message[:unapprove] = _('Unapproved the current merge request.')
end
diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb
index 89ec996488f..9f7599d2500 100644
--- a/lib/gitlab/redis.rb
+++ b/lib/gitlab/redis.rb
@@ -14,7 +14,6 @@ module Gitlab
Gitlab::Redis::FeatureFlag,
Gitlab::Redis::Queues,
Gitlab::Redis::QueuesMetadata,
- Gitlab::Redis::Pubsub,
Gitlab::Redis::RateLimiting,
Gitlab::Redis::RepositoryCache,
Gitlab::Redis::Sessions,
diff --git a/lib/gitlab/redis/cluster_util.rb b/lib/gitlab/redis/cluster_util.rb
index 5f1f39b5237..9e307940de3 100644
--- a/lib/gitlab/redis/cluster_util.rb
+++ b/lib/gitlab/redis/cluster_util.rb
@@ -26,6 +26,15 @@ module Gitlab
end
expired_count
end
+
+ # Redis cluster alternative to mget
+ def batch_get(keys, redis)
+ keys.each_slice(1000).flat_map do |subset|
+ Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline|
+ subset.map { |key| pipeline.get(key) }
+ end
+ end
+ end
end
end
end
diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb
index bbe5a8add4b..6acbf83df24 100644
--- a/lib/gitlab/redis/multi_store.rb
+++ b/lib/gitlab/redis/multi_store.rb
@@ -63,8 +63,12 @@ module Gitlab
hlen
hmget
hscan_each
+ llen
+ lrange
mapped_hmget
mget
+ pfcount
+ pttl
scan
scan_each
scard
@@ -72,20 +76,32 @@ module Gitlab
smembers
sscan
sscan_each
+ strlen
ttl
+ type
+ zcard
+ zcount
+ zrange
+ zrangebyscore
+ zrevrange
zscan_each
+ zscore
].freeze
WRITE_COMMANDS = %i[
+ decr
del
eval
expire
flushdb
hdel
+ hincrby
hset
incr
incrby
mapped_hmset
+ pfadd
+ pfmerge
publish
rpush
sadd
@@ -93,8 +109,15 @@ module Gitlab
set
setex
setnx
+ spop
srem
+ srem?
unlink
+ zadd
+ zpopmin
+ zrem
+ zremrangebyrank
+ zremrangebyscore
memory
].freeze
@@ -254,11 +277,27 @@ module Gitlab
#
# Let's define it explicitly instead of propagating it to method_missing
def close
- if use_primary_and_secondary_stores?
- [primary_store, secondary_store].map(&:close).first
+ if same_redis_store?
+ # if same_redis_store?, `use_primary_store_as_default?` returns false
+ # but we should avoid a feature-flag check in `.close` to avoid checking out
+ # an ActiveRecord connection during clean up.
+ secondary_store.close
else
- default_store.close
+ [primary_store, secondary_store].map(&:close).first
+ end
+ end
+
+ # blpop blocks until an element to be popped exist in the list or after a timeout.
+ def blpop(*args)
+ result = default_store.blpop(*args)
+ if !!result && use_primary_and_secondary_stores?
+ # special case to accommodate Gitlab::JobWaiter as blpop is only used in JobWaiter
+ # 1s should be sufficient wait time to account for delays between 1st and 2nd lpush
+ # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2520#note_1630893702
+ non_default_store.blpop(args.first, timeout: 1)
end
+
+ result
end
private
@@ -380,7 +419,7 @@ module Gitlab
end
def redis_store?(store)
- store.is_a?(::Redis) || store.is_a?(::Redis::Namespace)
+ store.is_a?(::Redis)
end
def validate_stores!
diff --git a/lib/gitlab/redis/pubsub.rb b/lib/gitlab/redis/pubsub.rb
deleted file mode 100644
index b5022f467a2..00000000000
--- a/lib/gitlab/redis/pubsub.rb
+++ /dev/null
@@ -1,13 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Redis
- class Pubsub < ::Gitlab::Redis::Wrapper
- class << self
- def config_fallback
- SharedState
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb
index fb3a143121b..d12d3e8c6aa 100644
--- a/lib/gitlab/redis/shared_state.rb
+++ b/lib/gitlab/redis/shared_state.rb
@@ -3,6 +3,12 @@
module Gitlab
module Redis
class SharedState < ::Gitlab::Redis::Wrapper
+ def self.redis
+ primary_store = ::Redis.new(ClusterSharedState.params)
+ secondary_store = ::Redis.new(params)
+
+ MultiStore.new(primary_store, secondary_store, store_name)
+ end
end
end
end
diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb
index 2bcf4769b5a..d5470bc0016 100644
--- a/lib/gitlab/redis/wrapper.rb
+++ b/lib/gitlab/redis/wrapper.rb
@@ -19,7 +19,7 @@ module Gitlab
InvalidPathError = Class.new(StandardError)
class << self
- delegate :params, :url, :store, to: :new
+ delegate :params, :url, :store, :encrypted_secrets, to: :new
def with
pool.with { |redis| yield redis }
@@ -110,6 +110,14 @@ module Gitlab
raw_config_hash[:sentinels]
end
+ def secret_file
+ if raw_config_hash[:secret_file].blank?
+ File.join(Settings.encrypted_settings['path'], 'redis.yaml.enc')
+ else
+ Settings.absolute(raw_config_hash[:secret_file])
+ end
+ end
+
def sentinels?
sentinels && !sentinels.empty?
end
@@ -118,22 +126,44 @@ module Gitlab
::Redis::Store::Factory.create(redis_store_options.merge(extras))
end
+ def encrypted_secrets
+ # In rake tasks, we have to populate the encrypted_secrets even if the
+ # file does not exist, as it is the job of one of those tasks to create
+ # the file. In other cases, like when being loaded as part of spinning
+ # up test environment via `scripts/setup-test-env`, we should gate on
+ # the presence of the specified secret file so that
+ # `Settings.encrypted`, which might not be loadable does not gets
+ # called.
+ Settings.encrypted(secret_file) if File.exist?(secret_file) || ::Gitlab::Runtime.rake?
+ end
+
private
def redis_store_options
config = raw_config_hash
config[:instrumentation_class] ||= self.class.instrumentation_class
- result = if config[:cluster].present?
- config[:db] = 0 # Redis Cluster only supports db 0
- config
+ decrypted_config = parse_encrypted_config(config)
+
+ result = if decrypted_config[:cluster].present?
+ decrypted_config[:db] = 0 # Redis Cluster only supports db 0
+ decrypted_config
else
- parse_redis_url(config)
+ parse_redis_url(decrypted_config)
end
parse_client_tls_options(result)
end
+ def parse_encrypted_config(encrypted_config)
+ encrypted_config.delete(:secret_file)
+
+ decrypted_secrets = encrypted_secrets&.config
+ encrypted_config.merge!(decrypted_secrets) if decrypted_secrets
+
+ encrypted_config
+ end
+
def parse_redis_url(config)
redis_url = config.delete(:url)
redis_uri = URI.parse(redis_url)
diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb
index 2fd9dc9fa09..6ac37986d5c 100644
--- a/lib/gitlab/regex.rb
+++ b/lib/gitlab/regex.rb
@@ -5,6 +5,7 @@ module Gitlab
extend self
extend MergeRequests
extend Packages
+ extend Packages::Protection::Rules
def project_name_regex
# The character range \p{Alnum} overlaps with \u{00A9}-\u{1f9ff}
diff --git a/lib/gitlab/regex/packages.rb b/lib/gitlab/regex/packages.rb
index 6b178933a25..a0038d39318 100644
--- a/lib/gitlab/regex/packages.rb
+++ b/lib/gitlab/regex/packages.rb
@@ -3,6 +3,8 @@
module Gitlab
module Regex
module Packages
+ include ::Gitlab::Utils::StrongMemoize
+
CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze
CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze
@@ -74,8 +76,10 @@ module Gitlab
maven_app_name_regex
end
- def npm_package_name_regex
- @npm_package_name_regex ||= %r{\A(?:@(#{Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX})/)?[-+\.\_a-zA-Z0-9]+\z}o
+ def npm_package_name_regex(other_accepted_chars = nil)
+ strong_memoize_with(:npm_package_name_regex, other_accepted_chars) do
+ %r{\A(?:@(#{Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX})/)?[-+\.\_a-zA-Z0-9#{other_accepted_chars}]+\z}
+ end
end
def npm_package_name_regex_message
diff --git a/lib/gitlab/regex/packages/protection/rules.rb b/lib/gitlab/regex/packages/protection/rules.rb
new file mode 100644
index 00000000000..383f26fe92d
--- /dev/null
+++ b/lib/gitlab/regex/packages/protection/rules.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Regex
+ module Packages
+ module Protection
+ module Rules
+ def protection_rules_npm_package_name_pattern_regex
+ @protection_rules_npm_package_name_pattern_regex ||= npm_package_name_regex('*')
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb
index 3a389d3363f..d5e80053772 100644
--- a/lib/gitlab/request_forgery_protection.rb
+++ b/lib/gitlab/request_forgery_protection.rb
@@ -6,7 +6,8 @@
module Gitlab
module RequestForgeryProtection
- class Controller < BaseActionController
+ # rubocop:disable Rails/ApplicationController
+ class Controller < ActionController::Base
protect_from_forgery with: :exception, prepend: true
def initialize
@@ -39,5 +40,6 @@ module Gitlab
rescue ActionController::InvalidAuthenticityToken
false
end
+ # rubocop:enable Rails/ApplicationController
end
end
diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb
deleted file mode 100644
index 36a3a491de6..00000000000
--- a/lib/gitlab/rugged_instrumentation.rb
+++ /dev/null
@@ -1,45 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module RuggedInstrumentation
- def self.query_time
- query_time = SafeRequestStore[:rugged_query_time] || 0
- query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION)
- end
-
- def self.add_query_time(duration)
- SafeRequestStore[:rugged_query_time] ||= 0
- SafeRequestStore[:rugged_query_time] += duration
- end
-
- def self.query_time_ms
- (self.query_time * 1000).round(2)
- end
-
- def self.query_count
- SafeRequestStore[:rugged_call_count] ||= 0
- end
-
- def self.increment_query_count
- SafeRequestStore[:rugged_call_count] ||= 0
- SafeRequestStore[:rugged_call_count] += 1
- end
-
- def self.active?
- SafeRequestStore.active?
- end
-
- def self.add_call_details(details)
- return unless Gitlab::PerformanceBar.enabled_for_request?
-
- Gitlab::SafeRequestStore[:rugged_call_details] ||= []
- Gitlab::SafeRequestStore[:rugged_call_details] << details
- end
-
- def self.list_call_details
- return [] unless Gitlab::PerformanceBar.enabled_for_request?
-
- Gitlab::SafeRequestStore[:rugged_call_details] || []
- end
- end
-end
diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb
index d06f414bd9a..fada3b84401 100644
--- a/lib/gitlab/search_results.rb
+++ b/lib/gitlab/search_results.rb
@@ -191,9 +191,7 @@ module Gitlab
unless default_project_filter
project_ids = project_ids_relation
- if Feature.enabled?(:search_issues_hide_archived_projects, current_user) && !filters[:include_archived]
- project_ids = project_ids.non_archived
- end
+ project_ids = project_ids.non_archived unless filters[:include_archived]
issues = issues.in_projects(project_ids)
.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/420046')
@@ -218,9 +216,7 @@ module Gitlab
unless default_project_filter
project_ids = project_ids_relation
- if Feature.enabled?(:search_merge_requests_hide_archived_projects, current_user) && !filters[:include_archived]
- project_ids = project_ids.non_archived
- end
+ project_ids = project_ids.non_archived unless filters[:include_archived]
merge_requests = merge_requests.of_projects(project_ids)
end
diff --git a/lib/gitlab/seeders/ci/catalog/resource_seeder.rb b/lib/gitlab/seeders/ci/catalog/resource_seeder.rb
new file mode 100644
index 00000000000..2971dabe044
--- /dev/null
+++ b/lib/gitlab/seeders/ci/catalog/resource_seeder.rb
@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Seeders
+ module Ci
+ module Catalog
+ class ResourceSeeder
+ # This is currently disabled until it gets fixed: https://gitlab.com/gitlab-org/gitlab/-/issues/429649
+ # Initializes the class
+ #
+ # @param [String] Path of the group to find
+ # @param [Integer] Number of resources to create
+ def initialize(group_path:, seed_count:)
+ @group = Group.find_by_full_path(group_path)
+ @seed_count = seed_count
+ @current_user = @group&.first_owner
+ end
+
+ def seed
+ if @group.nil?
+ warn 'ERROR: Group was not found.'
+ return
+ end
+
+ @seed_count.times do |i|
+ create_ci_catalog_resource(i)
+ end
+ end
+
+ private
+
+ def create_project(name, index)
+ project = ::Projects::CreateService.new(
+ @current_user,
+ description: "This is Catalog resource ##{index}",
+ name: name,
+ namespace_id: @group.id,
+ path: name,
+ visibility_level: @group.visibility_level
+ ).execute
+
+ if project.saved?
+ project
+ else
+ warn project.errors.full_messages.to_sentence
+ nil
+ end
+ end
+
+ def create_template_yml(project)
+ template_content = <<~YAML
+ spec:
+ inputs:
+ stage:
+ default: test
+ ---
+ component-job:
+ script: echo job 1
+ stage: $[[ inputs.stage ]]
+ YAML
+
+ project.repository.create_file(
+ @current_user,
+ 'template.yml',
+ template_content,
+ message: 'Add template.yml',
+ branch_name: project.default_branch_or_main
+ )
+ end
+
+ def create_readme(project, index)
+ project.repository.create_file(
+ @current_user,
+ '/README.md',
+ "## Component stuff #{index}",
+ message: 'Add README.md',
+ branch_name: project.default_branch_or_main
+ )
+ end
+
+ def create_ci_catalog(project)
+ result = ::Ci::Catalog::Resources::CreateService.new(project, @current_user).execute
+ if result.success?
+ result.payload
+ else
+ warn "Project '#{project.name}' could not be converted to a Catalog resource"
+ nil
+ end
+ end
+
+ def create_ci_catalog_resource(index)
+ name = "ci_seed_resource_#{index}"
+
+ if Project.find_by_name(name).present?
+ warn "Project '#{name}' already exists!"
+ return
+ end
+
+ project = create_project(name, index)
+
+ return unless project
+
+ create_readme(project, index)
+ create_template_yml(project)
+
+ return unless create_ci_catalog(project)
+
+ warn "Project '#{name}' was saved successfully!"
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
index a1363e7b6b2..10a69acc037 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb
@@ -21,6 +21,7 @@ module Gitlab
include Gitlab::Utils::StrongMemoize
DEFAULT_DUPLICATE_KEY_TTL = 6.hours
+ SHORT_DUPLICATE_KEY_TTL = 10.minutes
DEFAULT_STRATEGY = :until_executing
STRATEGY_NONE = :none
@@ -134,7 +135,7 @@ module Gitlab
jid != existing_jid
end
- def set_deduplicated_flag!(expiry = duplicate_key_ttl)
+ def set_deduplicated_flag!
return unless reschedulable?
with_redis { |redis| redis.eval(DEDUPLICATED_SCRIPT, keys: [cookie_key]) }
@@ -173,7 +174,7 @@ module Gitlab
end
def duplicate_key_ttl
- options[:ttl] || DEFAULT_DUPLICATE_KEY_TTL
+ options[:ttl] || default_duplicate_key_ttl
end
private
@@ -182,6 +183,12 @@ module Gitlab
attr_reader :queue_name, :job
attr_writer :existing_jid
+ def default_duplicate_key_ttl
+ return SHORT_DUPLICATE_KEY_TTL if Feature.enabled?(:reduce_duplicate_job_key_ttl)
+
+ DEFAULT_DUPLICATE_KEY_TTL
+ end
+
def worker_klass
@worker_klass ||= worker_class_name.to_s.safe_constantize
end
diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb
index b065190f656..e7ce837de29 100644
--- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb
+++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb
@@ -20,7 +20,7 @@ module Gitlab
if duplicate_job.idempotent?
duplicate_job.update_latest_wal_location!
- duplicate_job.set_deduplicated_flag!(expiry)
+ duplicate_job.set_deduplicated_flag!
Gitlab::SidekiqLogging::DeduplicationLogger.instance.deduplicated_log(
job, strategy_name, duplicate_job.options)
diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb
index a8b3683e09f..37a9ed37891 100644
--- a/lib/gitlab/sidekiq_middleware/server_metrics.rb
+++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb
@@ -25,11 +25,6 @@ module Gitlab
def metrics
metrics = {
- sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS),
- sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
- sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
- sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS),
- sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS),
sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'),
sidekiq_jobs_interrupted_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_interrupted_total, 'Sidekiq jobs interrupted'),
sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'),
@@ -43,9 +38,24 @@ module Gitlab
metrics[:sidekiq_jobs_completion_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_JOB_DURATION_BUCKETS)
metrics[:sidekiq_jobs_queue_duration_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_QUEUE_DURATION_BUCKETS)
metrics[:sidekiq_jobs_failed_total] = ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed')
+
+ # resource usage
+ metrics[:sidekiq_jobs_cpu_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS)
+ metrics[:sidekiq_jobs_db_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS)
+ metrics[:sidekiq_jobs_gitaly_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS)
+ metrics[:sidekiq_redis_requests_duration_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS)
+ metrics[:sidekiq_elasticsearch_requests_duration_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS)
else
- # The sum metric is still used in GitLab.com for dashboards
+ # These metrics are used in GitLab.com dashboards
metrics[:sidekiq_jobs_completion_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_completion_seconds_sum, 'Total of seconds to complete Sidekiq job')
+ metrics[:sidekiq_jobs_completion_count] = ::Gitlab::Metrics.counter(:sidekiq_jobs_completion_count, 'Number of Sidekiq jobs completed')
+
+ # resource usage sums
+ metrics[:sidekiq_jobs_cpu_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_cpu_seconds_sum, 'Total seconds this Sidekiq job spent on the CPU')
+ metrics[:sidekiq_jobs_db_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_db_seconds_sum, 'Total seconds of database time to run Sidekiq job')
+ metrics[:sidekiq_jobs_gitaly_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_gitaly_seconds_sum, 'Total seconds Gitaly time to run Sidekiq job')
+ metrics[:sidekiq_redis_requests_duration_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_redis_requests_duration_seconds_sum, 'Total duration in seconds that a Sidekiq job spent in requests to a Redis server')
+ metrics[:sidekiq_elasticsearch_requests_duration_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_duration_seconds_sum, 'Total duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server')
end
metrics
@@ -89,8 +99,9 @@ module Gitlab
# in metrics and can use them in the `ThreadsSampler` for setting a label
Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME
- labels = create_labels(worker.class, queue, job)
- instrument(job, labels) do
+ @job = job
+ @labels = create_labels(worker.class, queue, job)
+ instrument do
yield
end
end
@@ -99,8 +110,8 @@ module Gitlab
attr_reader :metrics
- def instrument(job, labels)
- queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job)
+ def instrument
+ @queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job)
@metrics[:sidekiq_jobs_queue_duration_seconds]&.observe(labels, queue_duration) if queue_duration
@@ -114,43 +125,33 @@ module Gitlab
@metrics[:sidekiq_jobs_interrupted_total].increment(labels, 1)
end
- job_succeeded = false
+ @job_succeeded = false
monotonic_time_start = Gitlab::Metrics::System.monotonic_time
job_thread_cputime_start = get_thread_cputime
begin
transaction = Gitlab::Metrics::BackgroundTransaction.new
transaction.run { yield }
- job_succeeded = true
+ @job_succeeded = true
ensure
monotonic_time_end = Gitlab::Metrics::System.monotonic_time
job_thread_cputime_end = get_thread_cputime
- monotonic_time = monotonic_time_end - monotonic_time_start
- job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start
+ @monotonic_time = monotonic_time_end - monotonic_time_start
+ @job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start
- # sidekiq_running_jobs, sidekiq_jobs_failed_total should not include the job_status label
@metrics[:sidekiq_running_jobs].increment(labels, -1)
- if Feature.enabled?(:emit_sidekiq_histogram_metrics, type: :ops)
- @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded
- else
- # we don't need job_status label here
- @metrics[:sidekiq_jobs_completion_seconds_sum].increment(labels, monotonic_time)
- end
+ @instrumentation = job[:instrumentation] || {}
+
+ record_resource_usage_counters
# job_status: done, fail match the job_status attribute in structured logging
labels[:job_status] = job_succeeded ? "done" : "fail"
- instrumentation = job[:instrumentation] || {}
- @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime)
- @metrics[:sidekiq_jobs_completion_seconds]&.observe(labels, monotonic_time)
+ record_histograms
- @metrics[:sidekiq_jobs_db_seconds].observe(labels, ActiveRecord::LogSubscriber.runtime / 1000)
- @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(instrumentation))
@metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(instrumentation))
- @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(instrumentation))
@metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(instrumentation))
- @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(instrumentation))
@metrics[:sidekiq_mem_total_bytes].set(labels, get_thread_memory_total_allocations(instrumentation))
with_load_balancing_settings(job) do |settings|
@@ -162,15 +163,50 @@ module Gitlab
@metrics[:sidekiq_load_balancing_count].increment(labels.merge(load_balancing_labels), 1)
end
- sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS)
- Gitlab::Metrics::SidekiqSlis.record_execution_apdex(sli_labels, monotonic_time) if job_succeeded
- Gitlab::Metrics::SidekiqSlis.record_execution_error(sli_labels, !job_succeeded)
- Gitlab::Metrics::SidekiqSlis.record_queueing_apdex(sli_labels, queue_duration) if queue_duration
+ @sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS)
+ record_execution_sli
+ record_queueing_sli
end
end
private
+ attr_reader :labels, :job, :queue_duration, :job_succeeded, :monotonic_time, :job_thread_cputime, :instrumentation, :sli_labels
+
+ def record_resource_usage_counters
+ if Feature.enabled?(:emit_sidekiq_histogram_metrics, type: :ops)
+ @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded
+ else
+ @metrics[:sidekiq_jobs_completion_seconds_sum].increment(labels, monotonic_time)
+ @metrics[:sidekiq_jobs_completion_count].increment(labels, 1)
+ @metrics[:sidekiq_jobs_cpu_seconds_sum].increment(labels, job_thread_cputime)
+ @metrics[:sidekiq_jobs_db_seconds_sum].increment(labels, ActiveRecord::LogSubscriber.runtime / 1000)
+ @metrics[:sidekiq_jobs_gitaly_seconds_sum].increment(labels, get_gitaly_time(instrumentation))
+ @metrics[:sidekiq_redis_requests_duration_seconds_sum].increment(labels, get_redis_time(instrumentation))
+ @metrics[:sidekiq_elasticsearch_requests_duration_seconds_sum].increment(labels, get_elasticsearch_time(instrumentation))
+ end
+ end
+
+ def record_histograms
+ @metrics[:sidekiq_jobs_cpu_seconds]&.observe(labels, job_thread_cputime)
+
+ @metrics[:sidekiq_jobs_completion_seconds]&.observe(labels, monotonic_time)
+
+ @metrics[:sidekiq_jobs_db_seconds]&.observe(labels, ActiveRecord::LogSubscriber.runtime / 1000)
+ @metrics[:sidekiq_jobs_gitaly_seconds]&.observe(labels, get_gitaly_time(instrumentation))
+ @metrics[:sidekiq_redis_requests_duration_seconds]&.observe(labels, get_redis_time(instrumentation))
+ @metrics[:sidekiq_elasticsearch_requests_duration_seconds]&.observe(labels, get_elasticsearch_time(instrumentation))
+ end
+
+ def record_queueing_sli
+ Gitlab::Metrics::SidekiqSlis.record_queueing_apdex(sli_labels, queue_duration) if queue_duration
+ end
+
+ def record_execution_sli
+ Gitlab::Metrics::SidekiqSlis.record_execution_apdex(sli_labels, monotonic_time) if job_succeeded
+ Gitlab::Metrics::SidekiqSlis.record_execution_error(sli_labels, !job_succeeded)
+ end
+
def with_load_balancing_settings(job)
keys = %w[load_balancing_strategy worker_data_consistency]
return unless keys.all? { |k| job.key?(k) }
diff --git a/lib/gitlab/sidekiq_middleware/skip_jobs.rb b/lib/gitlab/sidekiq_middleware/skip_jobs.rb
index 34ad843e8ee..56b150116a3 100644
--- a/lib/gitlab/sidekiq_middleware/skip_jobs.rb
+++ b/lib/gitlab/sidekiq_middleware/skip_jobs.rb
@@ -80,13 +80,20 @@ module Gitlab
end
health_check_attrs = worker_class.database_health_check_attrs
- job_base_model = Gitlab::Database.schemas_to_base_models[health_check_attrs[:gitlab_schema]].first
+
+ tables, schema = health_check_attrs.values_at(:tables, :gitlab_schema)
+
+ if health_check_attrs[:block].respond_to?(:call)
+ schema, tables = health_check_attrs[:block].call(job['args'], schema, tables)
+ end
+
+ job_base_model = Gitlab::Database.schemas_to_base_models[schema].first
health_context = Gitlab::Database::HealthStatus::Context.new(
DatabaseHealthStatusChecker.new(job['jid'], worker_class.name),
job_base_model.connection,
- health_check_attrs[:tables],
- health_check_attrs[:gitlab_schema]
+ tables,
+ schema
)
Gitlab::Database::HealthStatus.evaluate(health_context).any?(&:stop?)
diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb
index 778d278146d..ae4aca7ff92 100644
--- a/lib/gitlab/sidekiq_status.rb
+++ b/lib/gitlab/sidekiq_status.rb
@@ -94,8 +94,17 @@ module Gitlab
keys = job_ids.map { |jid| key_for(jid) }
- with_redis { |redis| redis.mget(*keys) }
- .map { |result| !result.nil? }
+ status = with_redis do |redis|
+ Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do
+ if Gitlab::Redis::ClusterUtil.cluster?(redis)
+ Gitlab::Redis::ClusterUtil.batch_get(keys, redis)
+ else
+ redis.mget(*keys)
+ end
+ end
+ end
+
+ status.map { |result| !result.nil? }
end
# Returns the JIDs that are completed
diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb
index f127e14243c..3bbcd59f45e 100644
--- a/lib/gitlab/tracking.rb
+++ b/lib/gitlab/tracking.rb
@@ -1,5 +1,8 @@
# frozen_string_literal: true
+# WARNING: This module has been deprecated and will be removed in the future
+# Use InternalEvents.track_event instead https://docs.gitlab.com/ee/development/internal_analytics/internal_event_instrumentation/index.html
+
module Gitlab
module Tracking
class << self
diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb
index 1b7dcaa5cf4..a9b8dc313d0 100644
--- a/lib/gitlab/url_builder.rb
+++ b/lib/gitlab/url_builder.rb
@@ -40,6 +40,8 @@ module Gitlab
note_url(object, **options)
when Release
instance.release_url(object, **options)
+ when Organizations::Organization
+ instance.organization_url(object, **options)
when Project
instance.project_url(object, **options)
when Snippet
diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb
index 7252283d1b9..941c2f793c4 100644
--- a/lib/gitlab/usage/metric_definition.rb
+++ b/lib/gitlab/usage/metric_definition.rb
@@ -3,7 +3,7 @@
module Gitlab
module Usage
class MetricDefinition
- METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json')
+ METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema', '**', '*.json')
AVAILABLE_STATUSES = %w[active broken].to_set.freeze
VALID_SERVICE_PING_STATUSES = %w[active broken].to_set.freeze
@@ -52,7 +52,7 @@ module Gitlab
end
def validate!
- self.class.schemer.validate(attributes.deep_stringify_keys).each do |error|
+ errors.each do |error|
error_message = <<~ERROR_MSG
Error type: #{error['type']}
Data: #{error['data']}
@@ -104,8 +104,10 @@ module Gitlab
definitions[key_path]&.to_context
end
- def schemer
- @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH))
+ def schemers
+ @schemers ||= Dir[METRIC_SCHEMA_PATH].map do |path|
+ ::JSONSchemer.schema(Pathname.new(path))
+ end
end
def dump_metrics_yaml
@@ -145,6 +147,19 @@ module Gitlab
private
+ def errors
+ result = []
+
+ self.class.schemers.each do |schemer|
+ # schemer.validate returns an Enumerator object
+ schemer.validate(attributes.deep_stringify_keys).each do |error|
+ result << error
+ end
+ end
+
+ result
+ end
+
def method_missing(method, *args)
attributes[method] || super
end
diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb
index b2027791e9d..5f819f060e4 100644
--- a/lib/gitlab/usage_data.rb
+++ b/lib/gitlab/usage_data.rb
@@ -174,7 +174,6 @@ module Gitlab
prometheus_enabled: alt_usage_data(fallback: nil) { Gitlab::Prometheus::Internal.prometheus_enabled? },
prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? },
reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::Email::IncomingEmail.enabled? },
- web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { false },
signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? },
grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? },
gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? }
diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
index f9dc8bd8a3c..185b49d4a68 100644
--- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb
+++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb
@@ -1,5 +1,8 @@
# frozen_string_literal: true
+# WARNING: This module has been deprecated and will be removed in the future
+# Use InternalEvents.track_event instead https://docs.gitlab.com/ee/development/internal_analytics/internal_event_instrumentation/index.html
+
module Gitlab
module UsageDataCounters
module HLLRedisCounter
@@ -53,8 +56,6 @@ module Gitlab
private
def track(values, event_name, time: Time.zone.now)
- return unless ::ServicePing::ServicePingSettings.enabled?
-
event = event_for(event_name)
Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present?
diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb
index 591e431c871..3f16681b642 100644
--- a/lib/gitlab/usage_data_counters/redis_counter.rb
+++ b/lib/gitlab/usage_data_counters/redis_counter.rb
@@ -1,17 +1,16 @@
# frozen_string_literal: true
+# WARNING: This module has been deprecated and will be removed in the future
+# Use InternalEvents.track_event instead https://docs.gitlab.com/ee/development/internal_analytics/internal_event_instrumentation/index.html
+
module Gitlab
module UsageDataCounters
module RedisCounter
def increment(redis_counter_key)
- return unless ::ServicePing::ServicePingSettings.enabled?
-
Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) }
end
def increment_by(redis_counter_key, incr)
- return unless ::ServicePing::ServicePingSettings.enabled?
-
Gitlab::Redis::SharedState.with { |redis| redis.incrby(redis_counter_key, incr) }
end
diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb
index f2db7e3c9b9..057e89a2a97 100644
--- a/lib/gitlab/workhorse.rb
+++ b/lib/gitlab/workhorse.rb
@@ -20,7 +20,7 @@ module Gitlab
include JwtAuthenticatable
class << self
- def git_http_ok(repository, repo_type, user, action, show_all_refs: false)
+ def git_http_ok(repository, repo_type, user, action, show_all_refs: false, need_audit: false)
raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s)
attrs = {
@@ -28,6 +28,7 @@ module Gitlab
GL_REPOSITORY: repo_type.identifier_for_container(repository.container),
GL_USERNAME: user&.username,
ShowAllRefs: show_all_refs,
+ NeedAudit: need_audit,
Repository: repository.gitaly_repository.to_h,
GitConfigOptions: [],
GitalyServer: {